One source now has several catalog revisions, so a publish naming branch and source alone does not say which pipeline published it. The seat mesh-builds' package-published gains an optional revision (seatgen regenerated). The version reserved for a build keeps the catalog revision number: from the picture's half when the picture starts the build (the same revision its source comes from), from the step ask's run when the ask reaches the build first. Its package-published says it. A reservation from before the field says none. Builds: design 47 H H-A-218, H-A-220
321 lines
12 KiB
Go
321 lines
12 KiB
Go
package build
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
)
|
|
|
|
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
|
|
// ask names its branch and source revision, and the builder reserves the version before it builds. When the builder's picture
|
|
// of that branch does not hold that source revision yet, or holds it but has not taken in the entry of the asked fingerprint
|
|
// from it, the ask waits for the picture to take it in, woken by each entry the picture takes in (take, EntryGone) and each
|
|
// revision its view holds (PictureHeld, H-A-162), never a
|
|
// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the
|
|
// branch, and nothing is built. A step ask with no source (a run on request) does not wait.
|
|
//
|
|
// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision
|
|
// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for
|
|
// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it
|
|
// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for.
|
|
|
|
// DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156).
|
|
const DefaultRevisionWait = 2 * time.Minute
|
|
|
|
func (b *Builder) revisionWait() time.Duration {
|
|
if b.RevisionWait > 0 {
|
|
return b.RevisionWait
|
|
}
|
|
return DefaultRevisionWait
|
|
}
|
|
|
|
// PictureHeld is told each revision a branch's view takes in whole, whether or not an entry changed (H-A-162): every step
|
|
// ask waiting for its revision looks again.
|
|
func (b *Builder) PictureHeld() {
|
|
b.init()
|
|
b.pictureMoved()
|
|
}
|
|
|
|
// pictureMoved wakes every step ask waiting for the picture: an entry was taken in or went, or a revision was held.
|
|
func (b *Builder) pictureMoved() {
|
|
b.mu.Lock()
|
|
moved := b.moved
|
|
b.moved = make(chan struct{})
|
|
b.mu.Unlock()
|
|
close(moved)
|
|
}
|
|
|
|
// holdsRevision says whether the builder's picture of branch holds the source revision and has taken in every entry of the
|
|
// ask from that branch.
|
|
func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuilds.BuildEntry) bool {
|
|
held, ok := b.Picture(branch)
|
|
if !ok || held < source {
|
|
return false
|
|
}
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
for _, e := range entries {
|
|
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
|
if err != nil {
|
|
return true // the build key is refused where the entry is built, with why
|
|
}
|
|
if b.origin[key].Branch != branch && b.heard[branch+"\x00"+e.Module].Fp2 != e.Fingerprint {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its
|
|
// source revision) and that revision names every entry of the ask; else failed or could not run, with its words.
|
|
func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) {
|
|
below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held)
|
|
if b.KeptRevision == nil || run.Revision <= 0 {
|
|
return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by"
|
|
}
|
|
source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision)
|
|
if err != nil {
|
|
b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err)
|
|
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err)
|
|
}
|
|
if source != run.Source {
|
|
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source)
|
|
}
|
|
for _, e := range run.Entries {
|
|
if fps[e.Module] != e.Fingerprint {
|
|
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
|
|
}
|
|
}
|
|
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
|
|
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
|
|
b.mu.Lock()
|
|
for _, e := range run.Entries {
|
|
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
|
if err != nil || b.origin[key].Branch != "" {
|
|
continue
|
|
}
|
|
first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module)
|
|
b.trunkFP[key] = true
|
|
b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first}
|
|
}
|
|
b.mu.Unlock()
|
|
b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision)
|
|
return "", ""
|
|
}
|
|
|
|
// awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the
|
|
// step's outcome failed, at the builder's stop could not run, each with its words.
|
|
func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) {
|
|
if b.Picture == nil || run.Source <= 0 || run.Branch == "" {
|
|
return "", ""
|
|
}
|
|
if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) {
|
|
return b.keptRevisionHolds(ctx, run, held)
|
|
}
|
|
bound := b.revisionWait()
|
|
notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch)
|
|
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
|
|
defer stop()
|
|
told := false
|
|
for {
|
|
b.mu.Lock()
|
|
moved := b.moved
|
|
b.mu.Unlock()
|
|
if b.holdsRevision(run.Branch, run.Source, run.Entries) {
|
|
if told {
|
|
b.logf("%s: revision %d of %s is in the builder's picture: the build goes on", run.Pipeline, run.Source, run.Branch)
|
|
}
|
|
return "", ""
|
|
}
|
|
if !told {
|
|
b.logf("%s: %s yet: the ask waits for it, at most %s", run.Pipeline, notHeld, bound)
|
|
told = true
|
|
}
|
|
select {
|
|
case <-moved:
|
|
case <-wait.Done():
|
|
if ctx.Err() != nil {
|
|
return meshbuilds.StepEndedOutcomeCouldNotRun, notHeld + ": the builder stopped before it was"
|
|
}
|
|
b.logf("%s: %s within %s: the step fails, nothing is built", run.Pipeline, notHeld, bound)
|
|
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notHeld, bound)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry
|
|
// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its
|
|
// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions,
|
|
// H-A-194), then waits for the registry seat's published of that package path in that series (the
|
|
// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to
|
|
// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a
|
|
// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and
|
|
// nothing is built.
|
|
|
|
// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default.
|
|
const DefaultPackageWait = 10 * time.Minute
|
|
|
|
func (b *Builder) packageWait() time.Duration {
|
|
if b.PackageWait > 0 {
|
|
return b.PackageWait
|
|
}
|
|
return DefaultPackageWait
|
|
}
|
|
|
|
// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the
|
|
// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package
|
|
// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing.
|
|
func (b *Builder) PackagePublished(path, version string) {
|
|
v := strings.TrimPrefix(version, "v")
|
|
series, ok := seriesOfVersion(v)
|
|
if path == "" || !ok {
|
|
return
|
|
}
|
|
b.init()
|
|
b.mu.Lock()
|
|
in := b.publishedPins[path]
|
|
if in == nil {
|
|
in = map[string]string{}
|
|
b.publishedPins[path] = in
|
|
}
|
|
if old := in[series]; old == "" || compareVersions(v, old) > 0 {
|
|
in[series] = v
|
|
}
|
|
b.mu.Unlock()
|
|
b.pictureMoved()
|
|
}
|
|
|
|
// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none.
|
|
func (b *Builder) publishedPin(path, series string) string {
|
|
b.mu.Lock()
|
|
v := b.publishedPins[path][series]
|
|
b.mu.Unlock()
|
|
if v == "" {
|
|
return ""
|
|
}
|
|
return "v" + v
|
|
}
|
|
|
|
// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest
|
|
// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and
|
|
// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops.
|
|
func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) {
|
|
var unpinned []string
|
|
for p, v := range packages {
|
|
if v == "" {
|
|
unpinned = append(unpinned, p)
|
|
}
|
|
}
|
|
if len(unpinned) == 0 {
|
|
return packages, "", ""
|
|
}
|
|
slices.Sort(unpinned)
|
|
if b.Manifests == nil {
|
|
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0])
|
|
}
|
|
man, err := b.Manifests.Manifest(ctx, digest)
|
|
if err != nil {
|
|
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err)
|
|
}
|
|
// What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it.
|
|
for _, p := range unpinned {
|
|
if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" {
|
|
continue
|
|
}
|
|
vs, err := b.PublishedVersions(ctx, p)
|
|
if err != nil {
|
|
b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err)
|
|
continue
|
|
}
|
|
for _, v := range vs {
|
|
b.PackagePublished(p, v)
|
|
}
|
|
}
|
|
bound := b.packageWait()
|
|
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
|
|
defer stop()
|
|
told := false
|
|
for {
|
|
b.mu.Lock()
|
|
moved := b.moved
|
|
b.mu.Unlock()
|
|
pins := map[string]string{}
|
|
missing, missingSeries := "", ""
|
|
for _, p := range unpinned {
|
|
series := man.AcceptedSeries(p)
|
|
pins[p] = b.publishedPin(p, series)
|
|
if pins[p] == "" && missing == "" {
|
|
missing, missingSeries = p, series
|
|
}
|
|
}
|
|
if missing == "" {
|
|
out := maps.Clone(packages)
|
|
maps.Copy(out, pins)
|
|
if told {
|
|
b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins)
|
|
}
|
|
return out, "", ""
|
|
}
|
|
notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries)
|
|
if !told {
|
|
b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound)
|
|
told = true
|
|
}
|
|
select {
|
|
case <-moved:
|
|
case <-wait.Done():
|
|
if ctx.Err() != nil {
|
|
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
|
|
}
|
|
b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound)
|
|
return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
|
|
}
|
|
}
|
|
}
|
|
|
|
// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form.
|
|
func seriesOfVersion(v string) (string, bool) {
|
|
if _, ok := versionParts(v); !ok {
|
|
return "", false
|
|
}
|
|
major, rest, _ := strings.Cut(v, ".")
|
|
minor, _, _ := strings.Cut(rest, ".")
|
|
return major + "." + minor, true
|
|
}
|
|
|
|
// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form.
|
|
func versionParts(v string) ([3]int, bool) {
|
|
var out [3]int
|
|
parts := strings.Split(v, ".")
|
|
if len(parts) != 3 {
|
|
return out, false
|
|
}
|
|
for i, p := range parts {
|
|
n, err := strconv.Atoi(p)
|
|
if err != nil || n < 0 {
|
|
return out, false
|
|
}
|
|
out[i] = n
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// compareVersions orders two MAJOR.MINOR.PATCH versions numerically.
|
|
func compareVersions(a, b string) int {
|
|
x, _ := versionParts(a)
|
|
y, _ := versionParts(b)
|
|
for i := range x {
|
|
if x[i] != y[i] {
|
|
return x[i] - y[i]
|
|
}
|
|
}
|
|
return 0
|
|
}
|