Files
mesh-delivery/modules/mesh-delivery-builder/internal/build/step_test.go
T
jschoubben d8a900dac0 Read what the registry seat holds before a build waits for an unpinned package (H-A-194)
The publishes the builder heard lived only in its memory, so a build waiting for a package published before a
restart waited the whole bound and failed although the registry held the version. Before it waits, the build asks
the Go package registry seat's versions once for each unpinned path and takes the newest in the accepted series
through PackagePublished; a failed read is said and the wait goes on by the published events.

Builds: design 47 H H-A-194, H-A-192, H-A-165
2026-11-20 21:48:40 +01:00

386 lines
18 KiB
Go

package build
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-builder/internal/versions"
)
// The runner's build step ask is the builder's one entry point for a candidate (design 47 G A-97): each entry is read by its
// fingerprint, built through the one claim, and the step's end says passed only for a build that is done.
func TestABuildStepAnswersItsEnd(t *testing.T) {
r := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
entryOf := func(fp, commit string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: commit, Fp: fp, ManifestDigest: mdigest('a'), Path: "."}
}
r.b.StepEntry = func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == fp1 {
return entryOf(fp1, "c1"), nil
}
return meshcatalog.CatalogEntry{}, errors.New("not found")
}
ask := func(entries ...meshbuilds.BuildEntry) meshbuilds.StepEnded {
return r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
}
if end := ask(); end.Outcome != meshbuilds.StepEndedOutcomeNothingToRun {
t.Errorf("an ask naming no entry: %+v", end)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: "src2:" + strings.Repeat("2", 64)}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "cannot be read") {
t.Errorf("an entry the seat does not hold: %+v", end)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: fp1, Commit: "other"}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun {
t.Errorf("an entry at another commit: %+v", end)
}
// A build already done is passed at once, naming each entry as it was asked.
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a')})
if _, err := r.st.Create(t.Context(), r.key, done); err != nil {
t.Fatal(err)
}
entries := []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1, Commit: "c1"}}
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
t.Errorf("a done build: %+v", end)
}
}
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
func TestAStepAskIsWokenByItsBuildsEnd(t *testing.T) {
r := newRig(t)
key := r.key
ch := r.b.awaitEnd(key)
r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildStartedEvent, Module: "m", FP: fp1}) // a start wakes nothing
select {
case <-ch:
t.Fatal("a build's start woke the wait")
default:
}
r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: fp1, Outcome: builds.BuildDone})
select {
case e := <-ch:
if e.Outcome != builds.BuildDone {
t.Errorf("end = %+v", e)
}
case <-time.After(time.Second):
t.Fatal("the end of the build did not wake the wait")
}
r.b.dropEnd(key, ch)
if len(r.b.stepWaits) != 0 {
t.Errorf("waits left: %v", r.b.stepWaits)
}
}
// A build step passes a build that reserves a version only when a version is reserved for its fingerprint (design 47 G A-456):
// a trunk build that ended done with none fails naming builder_reserve_done, and passes once one is reserved for it; a build
// that reserves none (a manifest of before G, version "1") passes as it did.
func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) {
for _, c := range []struct {
name, version string
reserve, released, unreadable bool
want, said string
}{
{"no version reserved", "0.1", false, false, false, meshbuilds.StepEndedOutcomeFailed, "built at " + fp1 + ", but no version is reserved for it"},
{"a version reserved", "0.1", true, false, false, meshbuilds.StepEndedOutcomePassed, ""},
{"a released build whose reservation is gone", "0.1", false, true, false, meshbuilds.StepEndedOutcomePassed, ""},
{"the plan cannot be read", "0.1", false, false, true, meshbuilds.StepEndedOutcomeCouldNotRun, "plan cannot be read"},
{"a series of before G", "1", false, false, false, meshbuilds.StepEndedOutcomePassed, ""},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.version, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}}
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { // built while the builder reserved nothing, as the live key was
t.Fatalf("build = %+v, want done", v)
}
r.b.Reserved = res
if c.reserve {
res.sources = append(res.sources, versions.Source{Fingerprint: fp1})
}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && !strings.Contains(end.Said, "builder_reserve_done")) {
t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said)
}
})
}
t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision)
t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails)
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage)
t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait)
}
// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it
// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish.
func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) {
var held atomic.Int64
r, _, entry := revisionRig(t, &held, make(chan struct{}, 1))
r.b.Picture = nil // no revision to wait for here
entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
waiting := make(chan struct{}, 1)
r.b.Logf = func(format string, args ...any) {
t.Logf(format, args...)
if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") {
select {
case waiting <- struct{}{}:
default:
}
}
}
return r, entry, waiting
}
const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x"
// awaitWaiting fails the test unless the build started its wait before ended said anything.
func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) {
t.Helper()
select {
case <-waiting:
case end := <-ended:
t.Fatalf("the build ended before the package was published: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the build never waited for the package's publish")
}
}
// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in
// another series and the module's release number pin nothing.
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _, waiting := unpinnedRig(t)
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package was published")
}
r.b.PackagePublished(unpinnedX, "v0.1.3")
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the ask")
}
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 ||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
}
}
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
!strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") {
t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built)
}
// A publish heard before gives the pin at once.
r.b.PackagePublished(unpinnedX, "v0.1.2")
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
t.Errorf("a publish heard did not pin the package: %+v", end)
}
}
// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series
// pins the package with no event heard; one only in another series leaves the build waiting for a publish.
func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin
var read []string
r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) {
read = append(read, path)
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) {
t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read)
}
r2, _, waiting := unpinnedRig(t)
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
r2.b.PackagePublished(unpinnedX, "0.1.10")
if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed {
t.Errorf("end = %+v, want passed once x was published in its series", end)
}
r2.b.Wait()
}
// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the
// key past the bound naming the package.
func aLaneWaitsForAnUnpinnedPackage(t *testing.T) {
r, entry, waiting := unpinnedRig(t)
done := make(chan struct{})
go func() {
r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry)
r.b.Wait()
close(done)
}()
awaitWaiting(t, waiting, done)
if asks := r.asks.Asks(); len(asks) != 0 {
t.Fatalf("a build was asked without the package's pin: %+v", asks)
}
r.b.PackagePublished(unpinnedX, "0.1.5")
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the lane")
}
if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" {
t.Errorf("asks %+v; want one asked with x at v0.1.5", asks)
}
// Past its bound the key fails naming the package, and nothing is asked.
r2, entry2, _ := unpinnedRig(t)
r2.b.PackageWait = time.Nanosecond
r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2)
r2.b.Wait()
if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") {
t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key])
}
}
func aRevisionHeldWakesTheAsk(t *testing.T) {
var held atomic.Int64
held.Store(40)
asked := make(chan struct{}, 1)
r, res, entry := revisionRig(t, &held, asked)
r.b.Entry(context.Background(), Origin{Branch: "main", Source: 40, First: true}, entry) // the entry, unchanged since 40
r.b.Wait()
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
case <-asked:
case end := <-ended:
t.Fatalf("the ask ended before the picture held its revision: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the ask never read the builder's picture")
}
held.Store(41)
r.b.PictureHeld() // revision 8, from source 41, changes no entry
select {
case end := <-ended:
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 {
t.Errorf("end = %+v, reserved from %+v; want passed with its version reserved", end, res.sources)
}
case <-time.After(5 * time.Second):
t.Fatal("a revision held did not wake the ask")
}
}
// revisionRig is a rig whose builder reserves, its step entry m at fp1, its picture of main holding the source revision held.
func revisionRig(t *testing.T, held *atomic.Int64, asked chan struct{}) (*rig, *fakeReservations, meshcatalog.CatalogEntry) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true}
r.b.Reserved = res
entry := meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog", Path: "modules/m", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1,
ManifestDigest: mdigest('a'), Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)}}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
r.b.Picture = func(branch string) (int64, bool) {
select {
case asked <- struct{}{}:
default:
}
return held.Load(), branch == "main" && held.Load() > 0
}
return r, res, entry
}
func aStepAskWaitsForItsRevision(t *testing.T) {
var held atomic.Int64
held.Store(40)
asked := make(chan struct{}, 1)
r, res, entry := revisionRig(t, &held, asked)
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
case <-asked: // the ask read the picture, which holds source revision 40
case end := <-ended:
t.Fatalf("the ask ended before the picture held its revision: %+v, reserved from %+v", end, res.sources)
case <-time.After(5 * time.Second):
t.Fatal("the ask never read the builder's picture")
}
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the ask built before the picture held its revision")
}
held.Store(41)
r.b.Entry(context.Background(), Origin{Branch: "main", Source: 41, First: true}, entry) // the picture takes revision 41 in
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the ask was not woken by the entry the picture took in")
}
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 {
t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources)
}
}
func aStepAskPastItsRevisionBoundFails(t *testing.T) {
var held atomic.Int64
held.Store(40)
r, res, _ := revisionRig(t, &held, make(chan struct{}, 1))
r.b.RevisionWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "revision 41 of main not in the builder's picture") || built || len(res.sources) != 0 {
t.Errorf("end = %+v, built %v, reserved %+v; want failed naming revision 41 of main, nothing built", end, built, res.sources)
}
// A step ask with no source (a run on request) does not wait (H-A-156).
end = r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if strings.Contains(end.Said, "not in the builder's picture") {
t.Errorf("an ask with no source waited: %+v", end)
}
}
// digestOf is the manifest digest a step's entry names: the rig's manifest, or one no store holds, whose plan cannot be read.
func digestOf(unreadable bool) string {
if unreadable {
return mdigest('z')
}
return mdigest('a')
}