package build import ( "context" "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) func haveValue(t *testing.T, phase delivery.Phase, health delivery.Health) []byte { t.Helper() b, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: health}) if err != nil { t.Fatal(err) } return b } // 1. A module of an archive only: the real indexer gives an archive no language (inputs.go languagesOf), so its // fingerprint names no toolchain, and the builder builds it on the build module's nodes with no toolchain agreed. func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *testing.T) { // review round 4, 1 r := newRig(t, archiveA) v := func() delivery.BuilderBuild { r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) r.b.Wait() if _, ok := r.st.vals[r.key]; !ok { t.Fatal("no build key was written for the archive-only module") } return valueOf(t, r.st, r.key) }() if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 { t.Fatalf("build = %+v, want done with the archive and no toolchain named", v) } if n := len(r.bm.callsOf("build_go_build")); n != 2 { t.Errorf("%d go builds, want one on each of two nodes", n) } } func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1 r := newRig(t, archiveA) r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait r.b.Health = &HaveSet{} // no build module is healthy anywhere r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") { t.Errorf("build = %+v, want failed: no node's build module is healthy", v) } have := &HaveSet{} have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) r.b.Health = have r.b.Reconsider(context.Background()) r.b.Wait() v := valueOf(t, r.st, r.key) if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix) { t.Errorf("build = %+v, want done on novox, the one node that qualifies, marked as one node", v) } } func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1 p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}}) if err != nil || !p.NeedsToolchain(GoBuildModule) { t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(GoBuildModule), err) } p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}}) if p.NeedsToolchain(GoBuildModule) { t.Error("a plan of an archive needs a toolchain") } } // 2. The digests-differ failure opens with its prefix, and nothing builds it again. func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testing.T) { // review round 4, 2 r := newRig(t, goBundle) r.bm.digests["ace/tool"] = "sha256:" + hex64('9') v := r.consider(t, "modules/m") if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, "digests differ:") { t.Fatalf("build = %+v, want a failure opening with exactly %q", v, "digests differ:") } calls, writes := len(r.bm.calls), r.st.writes r.bm.digests = map[string]string{} // the nodes would now agree r.b.Reconsider(context.Background()) r.b.Wait() r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if len(r.bm.calls) != calls || r.st.writes != writes { t.Errorf("a failed-for-digests key was built again: %d calls (was %d), %d writes (was %d)", len(r.bm.calls), calls, r.st.writes, writes) } } func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2 st := newMem() seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) writes := st.writes got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true}) if err != nil || got != nil || st.writes != writes { t.Errorf("claim = %v, %v, %d writes (was %d), want none and no write", got, err, st.writes, writes) } } // 3. A done key built on one node is built again on a second node once one qualifies. func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *testing.T) { // review round 4, 3 r := newRig(t, goBundle) r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") { t.Fatalf("build = %+v, want done on one node, marked", v) } if got := OneNodeModules(v); len(got) != 1 || got[0] != GoBuildModule { t.Errorf("OneNodeModules = %v", got) } rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build")) // Nothing changes while one node qualifies: looking again builds nothing. r.b.Reconsider(context.Background()) r.b.Wait() if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_go_build")) != calls { t.Fatal("a one-node key was built again with no second node") } // ace comes online: the key is built on ace and novox, as a new revision. r.pres.Put("ace", presenceValue(t, true), false) r.b.Reconsider(context.Background()) r.b.Wait() v = valueOf(t, r.st, r.key) if v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") { t.Errorf("build = %+v, want done on ace and novox", v) } if r.st.revs[r.key] <= rev { t.Errorf("revision %d, was %d: not a new revision", r.st.revs[r.key], rev) } builds := r.bm.callsOf("build_go_build") if len(builds) != calls+2 || builds[len(builds)-2].node == builds[len(builds)-1].node { t.Errorf("%d builds (was %d), want one on each of two nodes", len(builds), calls) } } func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.T) { // review round 4, 3 r := newRig(t, goBundle) r.pres.Put("ace", presenceValue(t, false), false) r.consider(t, "modules/m") r.bm.digests["ace/tool"] = "sha256:" + hex64('9') r.pres.Put("ace", presenceValue(t, true), false) r.b.Reconsider(context.Background()) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, DigestsDifferPrefix) { t.Errorf("build = %+v, want failed with digests differ", v) } } func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testing.T) { // review round 4, 3 r := newRig(t, goBundle) r.pres.Put("ace", presenceValue(t, false), false) first := r.consider(t, "modules/m") r.bm.fails["ace/build_go_fetch"] = "ace is out of disk" r.pres.Put("ace", presenceValue(t, true), false) r.b.Reconsider(context.Background()) r.b.Wait() if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) { t.Errorf("build = %+v, want the first done build to stand: %+v", v, first) } } // 4. A step never waits on a full node while another qualifying node has room. func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4 p := &Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}} entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks") total := 2 + 1 + 1 + 3 var waiting int for i := 1; i <= 4; i++ { // four modules choose before any lane has started room := 0 for _, n := range []string{"ace", "g14", "novox", "shanks"} { room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n] } nodes, _, err := ChooseNodes("go", entries, p, nil, 2) if err != nil || len(nodes) != 2 { t.Fatalf("choice %d: %v, %v", i, nodes, err) } if room >= 2 { for _, n := range nodes { if p.Reserved()[n] > p.CapacityOf(n) { t.Errorf("choice %d: %v put a lane on full %s while %d places were free", i, nodes, n, room) } } } else { waiting += 2 - room } } if want := 4*2 - total; waiting != want { t.Errorf("%d lanes wait, want %d: the 8 lanes less the %d places", waiting, want, total) } loads := p.Reserved() for n, c := range loads { if c > p.CapacityOf(n) && n != "shanks" && n != "ace" && n != "g14" && n != "novox" { t.Errorf("unknown node %s", n) } } if loads["shanks"] != 3 || loads["ace"] != 2 || loads["g14"]+loads["novox"] != 3 { t.Errorf("reservations = %v, want shanks 3, ace 2 and the single-place nodes taking the rest", loads) } } func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4 p := &Places{} nodes, _, _ := ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) if len(nodes) != 1 || p.Reserved()["ace"] != 1 { t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved()) } release, err := p.Acquire(context.Background(), "ace") if err != nil || p.Reserved()["ace"] != 0 || p.Running()["ace"] != 1 { t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running()) } release() ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) p.Cancel("ace") if len(p.Reserved()) != 0 { t.Errorf("reserved after Cancel = %v", p.Reserved()) } } // 5. A node whose build module is not healthy is not chosen. func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5 var s HaveSet if s.Healthy("ace", GoBuildModule) { t.Error("a module with no have is healthy") } if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", GoBuildModule) { t.Error("a running, healthy module is not") } if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) { t.Error("the same state counted as a change") } if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", GoBuildModule) { t.Error("a module whose health failed is healthy") } s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", GoBuildModule) { t.Error("a module whose phase failed is healthy") } s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", GoBuildModule) { t.Error("a deleted have is healthy") } if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) { t.Error("a node's key or a non-have counted") } } func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5 r := newRig(t, goBundle) have := &HaveSet{} have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) r.b.Health = have v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on one node, novox") { t.Fatalf("build = %+v, want done on novox alone", v) } for _, c := range r.bm.calls { if c.node == "ace" { t.Errorf("a step was called on ace, whose build module is unhealthy: %+v", c) } } have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) r.st.vals = map[string][]byte{} // a fresh key r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") { t.Errorf("build = %+v, want failed: no node has a healthy build module", v) } } // 6. A key found building is taken over, whoever holds it. func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6 st := newMem() seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.a"}) if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got == nil { t.Errorf("own building key: claim = %v, %v, want it taken again", got, err) } if got, err := (Claimer{Store: st, By: "ace.b"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true}); err != nil || got == nil || got.Build.By != "novox.a" { t.Errorf("another node's building key at start: claim = %v, %v, want it taken over", got, err) } if got, _ := (Claimer{Store: st, By: "ace.b"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); got != nil { t.Errorf("another builder's building key after the start was taken: a running builder's key is its own") } } func TestABuilderFindingAKeyBuildingByAnotherNodeBuildsItOnStart(t *testing.T) { // review round 4, 6 r := newRig(t, goBundle) seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD}) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || v.By != "novox.builder" { t.Errorf("build = %+v, want it taken over and done by novox.builder", v) } }