Design 47 H1: the builder fetches a module whole and never computes a pin (H-A-186, H-A-192, H-A-194) #291

Merged
mesh-tools merged 3 commits from 47-H1-builder-program-paths into main 2026-11-20 20:52:46 +00:00
13 changed files with 314 additions and 142 deletions
@@ -228,6 +228,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
b.PackageWait = time.Duration(packageWait) * time.Minute
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
b.PublishedVersions = publishedVersionsVia(calls.Call)
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error {
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
@@ -242,9 +244,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
b.Reserved = reserved
// A release heard, or the newest the builder's record keeps, pins a package an entry names with none (H-A-165).
reserved.heardReleased = b.VersionReleased
b.NewestReleased = reserved.r.NewestReleased
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
// A state bound later numbers the builds looked at without it.
@@ -44,15 +44,12 @@ type reservations struct {
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
built func(ctx context.Context, module, version string) error
released func(ctx context.Context, module, version string)
// heardReleased is told each version-released taken in (design 47 H H-A-165): a step ask waiting for a package of the
// module wakes. Nil tells nobody.
heardReleased func(module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// say says package-published on the seat mesh-builds; nil says none.
@@ -255,9 +252,6 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
if rv.released != nil {
rv.released(ctx, ev.Module, ev.Version)
}
if rv.heardReleased != nil {
rv.heardReleased(ev.Module, ev.Version)
}
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
default:
return false
@@ -134,15 +134,10 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
if err != nil || !strings.Contains(got.Summary, "m:b unpublished") || strings.Contains(got.Summary, "m:a unpublished") || !strings.Contains(got.Summary, "1.4.3") {
t.Fatalf("finding = %+v (%v), want version-unreleased naming m:b of 1.4.3", got, err)
}
var heardReleases []string // H-A-165: a release heard wakes a step ask waiting for a package of the module
rv.heardReleased = func(m, v string) { heardReleases = append(heardReleases, m+" "+v) }
released := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"1.4.3","source":1,"artifacts":[]}`)}
if !rv.heard(ctx, released) || !released.acked {
t.Fatalf("version-released was not taken in (acked %v)", released.acked)
}
if strings.Join(heardReleases, ", ") != "m 1.4.3" {
t.Errorf("the release heard was told %v, want m 1.4.3 (H-A-165)", heardReleases)
}
if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" {
t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items)
}
@@ -105,6 +105,10 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
if h.res != nil && se.event == versions.HeardPublished {
h.res.published(ctx, o)
}
if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil {
// A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192).
h.b.PackagePublished(o.Name, o.Version)
}
h.ack(m)
if len(wake) > 0 {
h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event)
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
@@ -96,6 +97,28 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e
return out, nil
}
// publishedVersionsVia asks the Go package registry seat's verb versions for one package path (design 47 H H-A-194): its
// versions, newest first.
func publishedVersionsVia(call toolCaller) func(context.Context, string) ([]string, error) {
return func(_ context.Context, path string) ([]string, error) {
raw, err := call("seat:"+gopackageregistry.Seat+"."+gopackageregistry.VerbVersions, gopackageregistry.VersionsInput{Name: path})
if err != nil {
return nil, err
}
var a gopackageregistry.VersionsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", gopackageregistry.Seat, err)
}
var out []string
for _, v := range a.Versions {
if v.Version != "" {
out = append(out, v.Version)
}
}
return out, nil
}
}
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
// versions, the seat git's tags and the requirements the catalog half's entries name (A-119). A call that fails is that source
// silent.
@@ -13,6 +13,7 @@ import (
func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f",
ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"},
Artifacts: []meshcatalog.Artifact{{Ref: "netcheck:lib", Kind: "package", Language: "go"}},
Inputs: &meshcatalog.EntryInputs{Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}},
Paths: map[string][]string{"lib": {"a/b", "c"}}}}
got := ModuleOf(e)
@@ -30,4 +31,11 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" {
t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare)
}
// A module with an artifact that names no paths (a Go program beside its packages) is built from its directory whole
// (ADR 0499 clauses 1 and 4): its fetch names no paths, so the program's go.mod and main package stay in the tree.
mixed := e
mixed.Artifacts = append([]meshcatalog.Artifact{{Ref: "netcheck:program", Kind: "bundle", Language: "go"}}, e.Artifacts...)
if got := ModuleOf(mixed); got.Inputs2 == nil || len(got.Inputs2.Paths) != 0 {
t.Errorf("ModuleOf of a module with a program naming no paths = %+v, want bases and no named paths", got.Inputs2)
}
}
@@ -5,11 +5,12 @@ import (
"fmt"
"maps"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
@@ -104,13 +105,16 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou
}
}
// **A package with no pin waits for its release** (design 47 H H-A-165, H-A-161): an entry keeps a mesh package none of
// whose versions is released with an empty version. Its step ask waits, as for its revision, for the release of the module
// that holds the package (the catalog's version-released, VersionReleased), up to PackageWait (the setting
// package-wait-minutes), then builds with that release as the package's pin (v<version>); past the bound the step fails,
// naming the package, and nothing is built.
// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry
// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its
// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions,
// H-A-194), then waits for the registry seat's published of that package path in that series (the
// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to
// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a
// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and
// nothing is built.
// DefaultPackageWait is the bound of a step ask's wait for a package's release, its setting package-wait-minutes' default.
// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default.
const DefaultPackageWait = 10 * time.Minute
func (b *Builder) packageWait() time.Duration {
@@ -120,91 +124,152 @@ func (b *Builder) packageWait() time.Duration {
return DefaultPackageWait
}
// VersionReleased takes the catalog's version-released of module in: a step ask waiting for a package of it wakes.
func (b *Builder) VersionReleased(module, version string) {
// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the
// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package
// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing.
func (b *Builder) PackagePublished(path, version string) {
v := strings.TrimPrefix(version, "v")
series, ok := seriesOfVersion(v)
if path == "" || !ok {
return
}
b.init()
b.mu.Lock()
b.releases[module] = version
in := b.publishedPins[path]
if in == nil {
in = map[string]string{}
b.publishedPins[path] = in
}
if old := in[series]; old == "" || compareVersions(v, old) > 0 {
in[series] = v
}
b.mu.Unlock()
b.pictureMoved()
}
// releaseOf is the pin a package is built against: the release of the module holding it, as Go names it; "" for none yet.
func (b *Builder) releaseOf(ctx context.Context, pkg string) string {
if b.Members == nil {
return ""
}
module, _, found, err := b.Members(ctx, pkg)
if err != nil || !found {
return ""
}
// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none.
func (b *Builder) publishedPin(path, series string) string {
b.mu.Lock()
v := b.releases[module]
v := b.publishedPins[path][series]
b.mu.Unlock()
if v == "" && b.NewestReleased != nil {
if kept, ok, err := b.NewestReleased(ctx, module); err == nil && ok {
v = kept
}
}
if v == "" {
return ""
}
return "v" + v
}
// awaitPins fills each package entry names with no pin from its release, waiting for one; no outcome once every package has
// a pin, failed past the bound, could not run at the builder's stop.
func (b *Builder) awaitPins(ctx context.Context, entry *meshcatalog.CatalogEntry) (outcome, said string) {
// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest
// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and
// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops.
func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) {
var unpinned []string
for p, v := range entry.Packages {
for p, v := range packages {
if v == "" {
unpinned = append(unpinned, p)
}
}
if len(unpinned) == 0 {
return "", ""
return packages, "", ""
}
slices.Sort(unpinned)
if b.Manifests == nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0])
}
man, err := b.Manifests.Manifest(ctx, digest)
if err != nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err)
}
// What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it.
for _, p := range unpinned {
if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" {
continue
}
vs, err := b.PublishedVersions(ctx, p)
if err != nil {
b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err)
continue
}
for _, v := range vs {
b.PackagePublished(p, v)
}
}
bound := b.packageWait()
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
defer stop()
pins := map[string]string{}
told := false
for {
b.mu.Lock()
moved := b.moved
b.mu.Unlock()
missing := ""
pins := map[string]string{}
missing, missingSeries := "", ""
for _, p := range unpinned {
if pins[p] == "" {
pins[p] = b.releaseOf(ctx, p)
}
series := man.AcceptedSeries(p)
pins[p] = b.publishedPin(p, series)
if pins[p] == "" && missing == "" {
missing = p
missing, missingSeries = p, series
}
}
if missing == "" {
packages := maps.Clone(entry.Packages)
maps.Copy(packages, pins)
entry.Packages = packages
out := maps.Clone(packages)
maps.Copy(out, pins)
if told {
b.logf("%s at %s: every package it names has a released version: built with %v", entry.Module, entry.Fp, pins)
b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins)
}
return "", ""
return out, "", ""
}
notReleased := fmt.Sprintf("package %s has no released version", missing)
notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries)
if !told {
b.logf("%s at %s: %s yet: the ask waits for its release, at most %s", entry.Module, entry.Fp, notReleased, bound)
b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound)
told = true
}
select {
case <-moved:
case <-wait.Done():
if ctx.Err() != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
}
b.logf("%s at %s: %s within %s: the step fails, nothing is built", entry.Module, entry.Fp, notReleased, bound)
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound)
return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
}
}
}
// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form.
func seriesOfVersion(v string) (string, bool) {
if _, ok := versionParts(v); !ok {
return "", false
}
major, rest, _ := strings.Cut(v, ".")
minor, _, _ := strings.Cut(rest, ".")
return major + "." + minor, true
}
// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two MAJOR.MINOR.PATCH versions numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
@@ -113,11 +113,13 @@ type Builder struct {
// RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait
// when 0.
RevisionWait time.Duration
// PackageWait bounds a step ask's wait for the release of a package its entry names with no pin, its setting
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. NewestReleased reads the newest version the builder's
// record of releases keeps for a module; nil reads none, and only a release heard (VersionReleased) gives a pin.
PackageWait time.Duration
NewestReleased func(ctx context.Context, module string) (version string, found bool, err error)
// PackageWait bounds a build's wait for the publish of a package its entry names with no pin, its setting
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
// gives a pin (H-A-192).
PackageWait time.Duration
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
// (H-A-194): read once before a build waits for that package. Nil reads none.
PublishedVersions func(ctx context.Context, path string) ([]string, error)
// Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine
// answered when last asked, and counted, every node the assignments names, present or away, a node with no statement
// stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in
@@ -162,8 +164,9 @@ type Builder struct {
builds map[string]context.CancelCauseFunc
// moved is closed and made again at each entry the picture takes in or drops: the step asks waiting for a revision wake.
moved chan struct{}
// releases are the versions heard released, by module (VersionReleased): a package with no pin waits for one.
releases map[string]string
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
// a package with no pin waits for one (H-A-192).
publishedPins map[string]map[string]string
}
type planned struct {
@@ -194,7 +197,7 @@ func (b *Builder) init() {
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
b.builds = map[string]context.CancelCauseFunc{}
b.moved = make(chan struct{})
b.releases = map[string]string{}
b.publishedPins = map[string]map[string]string{}
})
}
@@ -285,7 +288,7 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
// built against, its packages and needs, and the named paths of its artifacts.
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
@@ -294,7 +297,10 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
}
if in := e.Inputs; in != nil {
m.Toolchains, m.Problems = in.Toolchains, in.Problems
if len(in.Paths) > 0 {
// Only a module whose every artifact names its paths is built from those alone (ADR 0499 clauses 1, 2 and 4): an
// artifact without paths, such as a Go program, keeps its module's directory as its build source, so a module
// with one names no paths and its fetch keeps the directory whole.
if len(in.Paths) > 0 && everyArtifactNamesPaths(e) {
if m.Inputs2 == nil {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{}
}
@@ -309,6 +315,20 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
return m
}
// everyArtifactNamesPaths is whether the entry names artifacts and the named paths of each, by its reference or its name.
func everyArtifactNamesPaths(e meshcatalog.CatalogEntry) bool {
if len(e.Artifacts) == 0 || e.Inputs == nil {
return false
}
for _, a := range e.Artifacts {
_, name, _ := strings.Cut(a.Ref, ":")
if len(e.Inputs.Paths[a.Ref]) == 0 && len(e.Inputs.Paths[name]) == 0 {
return false
}
}
return true
}
// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build
// of the trunk's value was before branches were watched.
func (b *Builder) originOf(key string) Origin {
@@ -1112,9 +1132,15 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: b.OwnModule, At: time.Now(), Version: l.version, Build: job}
if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages)
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
// them into the workspace, whatever the repository's go.mod names. A package the entry names with no pin waits for
// its publish and is built with the version that publish carries; past the bound the lane fails naming it, and no
// ask is put (H-A-192, H-A-165).
pinned, outcome, said := b.awaitPublished(ctx, module, m.Fp2, m.Manifest, m.Packages)
if outcome != "" {
l.err = errors.New(said)
return
}
ask.Packages = maps.Clone(pinned)
for _, fb := range req.Builds {
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
}
@@ -333,8 +333,10 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil // no node has pulled it yet
// Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
// released, design 47 H-A-161) is no pin.
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
// published, design 47 H-A-161) is pinned at the version the registry seat's published carries (H-A-192), never
// asked without it.
r.b.PackagePublished("git.novox.be/novox/mesh-contracts/go/seats/x", "v0.1.7")
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.7"}
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
@@ -63,10 +63,12 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit)
}
// A package the entry names with no pin is built against its release, waited for (H-A-165).
if outcome, said := b.awaitPins(ctx, &entry); outcome != "" {
// A package the entry names with no pin is built against its publish, waited for (H-A-192, H-A-165).
pinned, outcome, said := b.awaitPublished(ctx, entry.Module, entry.Fp, meshcatalog.BareManifestDigest(entry.ManifestDigest), entry.Packages)
if outcome != "" {
return outcome, said
}
entry.Packages = pinned
key, err := builds.BuilderBuildKey(entry.Module, entry.Fp)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error()
@@ -3,9 +3,10 @@ package build
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -121,76 +122,155 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage)
t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait)
}
// unpinnedRig is a rig whose step entry m names the package x with no version, built by the module contracts.
func unpinnedRig(t *testing.T) (*rig, *fakeReservations) {
// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it
// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish.
func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) {
var held atomic.Int64
r, res, entry := revisionRig(t, &held, make(chan struct{}, 1))
r, _, entry := revisionRig(t, &held, make(chan struct{}, 1))
r.b.Picture = nil // no revision to wait for here
entry.Packages = map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/x": "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
return "contracts", "contracts:x", pkg == "git.novox.be/novox/mesh-contracts/go/seats/x", nil
waiting := make(chan struct{}, 1)
r.b.Logf = func(format string, args ...any) {
t.Logf(format, args...)
if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") {
select {
case waiting <- struct{}{}:
default:
}
}
}
return r, res
return r, entry, waiting
}
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _ := unpinnedRig(t)
var once sync.Once
looked := make(chan struct{})
r.b.NewestReleased = func(context.Context, string) (string, bool, error) { // the record holds none yet
once.Do(func() { close(looked) })
return "", false, nil
const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x"
// awaitWaiting fails the test unless the build started its wait before ended said anything.
func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) {
t.Helper()
select {
case <-waiting:
case end := <-ended:
t.Fatalf("the build ended before the package was published: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the build never waited for the package's publish")
}
}
// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in
// another series and the module's release number pin nothing.
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _, waiting := unpinnedRig(t)
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
case <-looked:
case end := <-ended:
t.Fatalf("the ask ended before the package was released: %+v, asks %+v", end, r.asks.Asks())
case <-time.After(5 * time.Second):
t.Fatal("the ask never looked for the package's release")
}
awaitWaiting(t, waiting, ended)
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package had a release")
t.Fatal("the build was made before the package was published")
}
r.b.VersionReleased("contracts", "0.1.3")
r.b.PackagePublished(unpinnedX, "v0.1.3")
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the release did not wake the ask")
t.Fatal("the publish did not wake the ask")
}
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 ||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{"git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
}
}
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
r, _ := unpinnedRig(t)
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
!strings.Contains(end.Said, "package git.novox.be/novox/mesh-contracts/go/seats/x has no released version") {
!strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") {
t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built)
}
// A release the builder's own record keeps gives the pin at once.
r.b.NewestReleased = func(_ context.Context, module string) (string, bool, error) {
return "0.1.2", module == "contracts", nil
}
// A publish heard before gives the pin at once.
r.b.PackagePublished(unpinnedX, "v0.1.2")
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
t.Errorf("a release the record keeps did not pin the package: %+v", end)
t.Errorf("a publish heard did not pin the package: %+v", end)
}
}
// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series
// pins the package with no event heard; one only in another series leaves the build waiting for a publish.
func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin
var read []string
r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) {
read = append(read, path)
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) {
t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read)
}
r2, _, waiting := unpinnedRig(t)
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
r2.b.PackagePublished(unpinnedX, "0.1.10")
if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed {
t.Errorf("end = %+v, want passed once x was published in its series", end)
}
r2.b.Wait()
}
// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the
// key past the bound naming the package.
func aLaneWaitsForAnUnpinnedPackage(t *testing.T) {
r, entry, waiting := unpinnedRig(t)
done := make(chan struct{})
go func() {
r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry)
r.b.Wait()
close(done)
}()
awaitWaiting(t, waiting, done)
if asks := r.asks.Asks(); len(asks) != 0 {
t.Fatalf("a build was asked without the package's pin: %+v", asks)
}
r.b.PackagePublished(unpinnedX, "0.1.5")
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the lane")
}
if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" {
t.Errorf("asks %+v; want one asked with x at v0.1.5", asks)
}
// Past its bound the key fails naming the package, and nothing is asked.
r2, entry2, _ := unpinnedRig(t)
r2.b.PackageWait = time.Nanosecond
r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2)
r2.b.Wait()
if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") {
t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key])
}
}
@@ -295,23 +295,6 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string)
return false, nil
}
// NewestReleased is the newest version the builder's own record of releases keeps for module (design 47 H H-A-165): a
// package with no pin is built against it; false when the record holds none.
func (r *Reserver) NewestReleased(ctx context.Context, module string) (string, bool, error) {
if r.Releases == nil {
return "", false, nil
}
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil || !found {
return "", false, err
}
var rec ReleasedRecord
if json.Unmarshal(raw, &rec) != nil || len(rec.Kept) == 0 {
return "", false, nil
}
return rec.Kept[len(rec.Kept)-1].Version, true, nil
}
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error {
n := r.ReleasedKept
@@ -201,13 +201,11 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
t.Errorf("reservations left: %v", all)
}
// One key per module holds the last releases, the oldest dropped (A-456b).
newest := ""
for _, fp := range []string{"src2:b1", "src2:b2"} {
n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp})
if err != nil {
t.Fatal(err)
}
newest = n.Version
if _, err := r.Released(ctx, "m", n.Version); err != nil {
t.Fatal(err)
}
@@ -221,13 +219,6 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok {
t.Error("the newest release is not kept")
}
// H-A-165: the newest version the record keeps is the pin of a package with none.
if v, ok, err := r.NewestReleased(ctx, "m"); !ok || err != nil || v != newest {
t.Errorf("newest released of m = %q, %v (%v), want %s", v, ok, err, newest)
}
if _, ok, _ := r.NewestReleased(ctx, "other"); ok {
t.Error("a module never released has a newest release")
}
}
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces