Files
mesh-delivery/modules/mesh-delivery-builder/internal/build/step_test.go
T
jschoubben e29aa8aca9 Design 47 H1: a done build key keeps the pins it was built with, and its step's end reports those (H-A-207)
A build step whose key was done before reported the pins it resolved at
that moment, so a publish since would have the runner ask the waiting
test steps against a version the build was never built with. The done
key's record keeps the versions its Go ask was built with, beside its
digests, and the step's end reports the record's; a done key keeping none
reports none, so the runner ends those steps could not run, loud.

Builds: design 47 H H-A-207, H-A-203
2026-11-20 23:12:29 +01:00

412 lines
20 KiB
Go

package build
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-builder/internal/versions"
)
// The runner's build step ask is the builder's one entry point for a candidate (design 47 G A-97): each entry is read by its
// fingerprint, built through the one claim, and the step's end says passed only for a build that is done.
func TestABuildStepAnswersItsEnd(t *testing.T) {
r := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
entryOf := func(fp, commit string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: commit, Fp: fp, ManifestDigest: mdigest('a'), Path: "."}
}
r.b.StepEntry = func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == fp1 {
return entryOf(fp1, "c1"), nil
}
return meshcatalog.CatalogEntry{}, errors.New("not found")
}
ask := func(entries ...meshbuilds.BuildEntry) meshbuilds.StepEnded {
return r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
}
if end := ask(); end.Outcome != meshbuilds.StepEndedOutcomeNothingToRun {
t.Errorf("an ask naming no entry: %+v", end)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: "src2:" + strings.Repeat("2", 64)}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "cannot be read") {
t.Errorf("an entry the seat does not hold: %+v", end)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: fp1, Commit: "other"}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun {
t.Errorf("an entry at another commit: %+v", end)
}
// A build already done is passed at once, naming each entry as it was asked.
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a')})
if _, err := r.st.Create(t.Context(), r.key, done); err != nil {
t.Fatal(err)
}
entries := []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1, Commit: "c1"}}
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
t.Errorf("a done build: %+v", end)
}
}
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
func TestAStepAskIsWokenByItsBuildsEnd(t *testing.T) {
r := newRig(t)
key := r.key
ch := r.b.awaitEnd(key)
r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildStartedEvent, Module: "m", FP: fp1}) // a start wakes nothing
select {
case <-ch:
t.Fatal("a build's start woke the wait")
default:
}
r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: fp1, Outcome: builds.BuildDone})
select {
case e := <-ch:
if e.Outcome != builds.BuildDone {
t.Errorf("end = %+v", e)
}
case <-time.After(time.Second):
t.Fatal("the end of the build did not wake the wait")
}
r.b.dropEnd(key, ch)
if len(r.b.stepWaits) != 0 {
t.Errorf("waits left: %v", r.b.stepWaits)
}
}
// A build step passes a build that reserves a version only when a version is reserved for its fingerprint (design 47 G A-456):
// a trunk build that ended done with none fails naming builder_reserve_done, and passes once one is reserved for it; a build
// that reserves none (a manifest of before G, version "1") passes as it did.
func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) {
for _, c := range []struct {
name, version string
reserve, released, unreadable bool
want, said string
}{
{"no version reserved", "0.1", false, false, false, meshbuilds.StepEndedOutcomeFailed, "built at " + fp1 + ", but no version is reserved for it"},
{"a version reserved", "0.1", true, false, false, meshbuilds.StepEndedOutcomePassed, ""},
{"a released build whose reservation is gone", "0.1", false, true, false, meshbuilds.StepEndedOutcomePassed, ""},
{"the plan cannot be read", "0.1", false, false, true, meshbuilds.StepEndedOutcomeCouldNotRun, "plan cannot be read"},
{"a series of before G", "1", false, false, false, meshbuilds.StepEndedOutcomePassed, ""},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.version, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}}
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { // built while the builder reserved nothing, as the live key was
t.Fatalf("build = %+v, want done", v)
}
r.b.Reserved = res
if c.reserve {
res.sources = append(res.sources, versions.Source{Fingerprint: fp1})
}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && !strings.Contains(end.Said, "builder_reserve_done")) {
t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said)
}
})
}
t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision)
t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails)
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage)
t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait)
}
// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it
// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish.
func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) {
var held atomic.Int64
r, _, entry := revisionRig(t, &held, make(chan struct{}, 1))
r.b.Picture = nil // no revision to wait for here
entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
waiting := make(chan struct{}, 1)
r.b.Logf = func(format string, args ...any) {
t.Logf(format, args...)
if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") {
select {
case waiting <- struct{}{}:
default:
}
}
}
return r, entry, waiting
}
const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x"
// awaitWaiting fails the test unless the build started its wait before ended said anything.
func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) {
t.Helper()
select {
case <-waiting:
case end := <-ended:
t.Fatalf("the build ended before the package was published: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the build never waited for the package's publish")
}
}
// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in
// another series and the module's release number pin nothing.
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _, waiting := unpinnedRig(t)
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package was published")
}
r.b.PackagePublished(unpinnedX, "v0.1.3")
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the ask")
}
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 ||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
}
// The end reports the pins the build was built with, the one the publish gave included (H-A-207).
if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages)
}
// A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207).
r.b.PackagePublished(unpinnedX, "v0.1.5")
again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
}
// A done key that keeps no pins reports none, so the runner ends the waiting steps loud.
v, err := builds.DecodeBuilderBuild(r.st.vals[r.key])
if err != nil {
t.Fatal(err)
}
v.Packages = nil
raw, err := builds.EncodeBuilderBuild(v)
if err != nil {
t.Fatal(err)
}
r.st.vals[r.key] = raw
if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 {
t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages)
}
}
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
!strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") {
t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built)
}
// A publish heard before gives the pin at once.
r.b.PackagePublished(unpinnedX, "v0.1.2")
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
t.Errorf("a publish heard did not pin the package: %+v", end)
}
}
// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series
// pins the package with no event heard; one only in another series leaves the build waiting for a publish.
func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin
var read []string
r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) {
read = append(read, path)
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) {
t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read)
}
r2, _, waiting := unpinnedRig(t)
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
r2.b.PackagePublished(unpinnedX, "0.1.10")
if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed {
t.Errorf("end = %+v, want passed once x was published in its series", end)
}
r2.b.Wait()
}
// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the
// key past the bound naming the package.
func aLaneWaitsForAnUnpinnedPackage(t *testing.T) {
r, entry, waiting := unpinnedRig(t)
done := make(chan struct{})
go func() {
r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry)
r.b.Wait()
close(done)
}()
awaitWaiting(t, waiting, done)
if asks := r.asks.Asks(); len(asks) != 0 {
t.Fatalf("a build was asked without the package's pin: %+v", asks)
}
r.b.PackagePublished(unpinnedX, "0.1.5")
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the lane")
}
if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" {
t.Errorf("asks %+v; want one asked with x at v0.1.5", asks)
}
// Past its bound the key fails naming the package, and nothing is asked.
r2, entry2, _ := unpinnedRig(t)
r2.b.PackageWait = time.Nanosecond
r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2)
r2.b.Wait()
if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") {
t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key])
}
}
func aRevisionHeldWakesTheAsk(t *testing.T) {
var held atomic.Int64
held.Store(40)
asked := make(chan struct{}, 1)
r, res, entry := revisionRig(t, &held, asked)
r.b.Entry(context.Background(), Origin{Branch: "main", Source: 40, First: true}, entry) // the entry, unchanged since 40
r.b.Wait()
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
case <-asked:
case end := <-ended:
t.Fatalf("the ask ended before the picture held its revision: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the ask never read the builder's picture")
}
held.Store(41)
r.b.PictureHeld() // revision 8, from source 41, changes no entry
select {
case end := <-ended:
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 {
t.Errorf("end = %+v, reserved from %+v; want passed with its version reserved", end, res.sources)
}
case <-time.After(5 * time.Second):
t.Fatal("a revision held did not wake the ask")
}
}
// revisionRig is a rig whose builder reserves, its step entry m at fp1, its picture of main holding the source revision held.
func revisionRig(t *testing.T, held *atomic.Int64, asked chan struct{}) (*rig, *fakeReservations, meshcatalog.CatalogEntry) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true}
r.b.Reserved = res
entry := meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog", Path: "modules/m", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1,
ManifestDigest: mdigest('a'), Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)}}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
r.b.Picture = func(branch string) (int64, bool) {
select {
case asked <- struct{}{}:
default:
}
return held.Load(), branch == "main" && held.Load() > 0
}
return r, res, entry
}
func aStepAskWaitsForItsRevision(t *testing.T) {
var held atomic.Int64
held.Store(40)
asked := make(chan struct{}, 1)
r, res, entry := revisionRig(t, &held, asked)
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
case <-asked: // the ask read the picture, which holds source revision 40
case end := <-ended:
t.Fatalf("the ask ended before the picture held its revision: %+v, reserved from %+v", end, res.sources)
case <-time.After(5 * time.Second):
t.Fatal("the ask never read the builder's picture")
}
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the ask built before the picture held its revision")
}
held.Store(41)
r.b.Entry(context.Background(), Origin{Branch: "main", Source: 41, First: true}, entry) // the picture takes revision 41 in
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the ask was not woken by the entry the picture took in")
}
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 {
t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources)
}
}
func aStepAskPastItsRevisionBoundFails(t *testing.T) {
var held atomic.Int64
held.Store(40)
r, res, _ := revisionRig(t, &held, make(chan struct{}, 1))
r.b.RevisionWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "revision 41 of main not in the builder's picture") || built || len(res.sources) != 0 {
t.Errorf("end = %+v, built %v, reserved %+v; want failed naming revision 41 of main, nothing built", end, built, res.sources)
}
// A step ask with no source (a run on request) does not wait (H-A-156).
end = r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if strings.Contains(end.Said, "not in the builder's picture") {
t.Errorf("an ask with no source waited: %+v", end)
}
}
// digestOf is the manifest digest a step's entry names: the rig's manifest, or one no store holds, whose plan cannot be read.
func digestOf(unreadable bool) string {
if unreadable {
return mdigest('z')
}
return mdigest('a')
}