Review round 1: an ask names the revision and the source of one revision, so a kept revision read by number whose source differs from the ask's is not the ask's; the check is restored, naming both sources. And a kept revision naming another source or other fingerprints is a mismatch no wait can mend: the step fails at once, loud, instead of waiting out revision-wait-minutes; only a revision the catalog does not answer is waited for. Builds: design 47 H-A-222 (2), H-A-218
343 lines
14 KiB
Go
343 lines
14 KiB
Go
package build
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
)
|
|
|
|
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
|
|
// ask names its branch and source revision, and the builder reserves the version before it builds. When the builder's picture
|
|
// of that branch does not hold that source revision yet, or holds it but has not taken in the entry of the asked fingerprint
|
|
// from it, the ask waits for the picture to take it in, woken by each entry the picture takes in (take, EntryGone) and each
|
|
// revision its view holds (PictureHeld, H-A-162), never a
|
|
// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the
|
|
// branch, and nothing is built. A step ask with no source (a run on request) does not wait.
|
|
//
|
|
// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision
|
|
// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for
|
|
// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it
|
|
// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for.
|
|
//
|
|
// **An ask naming a revision is held by its number** (design 47 H H-A-222): when the picture does not name the ask's
|
|
// entries, the ask's catalog revision is read by number, and the ask is built when the catalog keeps it and it names every
|
|
// fingerprint asked, whatever the source the picture holds says (a re-pin keeps a later revision of the same source, H-A-211,
|
|
// H-A-213, so the source alone cannot say it). The source is compared only for an ask naming no revision; an ask whose
|
|
// revision the catalog does not answer and whose source is not below the picture's waits as before.
|
|
|
|
// DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156).
|
|
const DefaultRevisionWait = 2 * time.Minute
|
|
|
|
func (b *Builder) revisionWait() time.Duration {
|
|
if b.RevisionWait > 0 {
|
|
return b.RevisionWait
|
|
}
|
|
return DefaultRevisionWait
|
|
}
|
|
|
|
// PictureHeld is told each revision a branch's view takes in whole, whether or not an entry changed (H-A-162): every step
|
|
// ask waiting for its revision looks again.
|
|
func (b *Builder) PictureHeld() {
|
|
b.init()
|
|
b.pictureMoved()
|
|
}
|
|
|
|
// pictureMoved wakes every step ask waiting for the picture: an entry was taken in or went, or a revision was held.
|
|
func (b *Builder) pictureMoved() {
|
|
b.mu.Lock()
|
|
moved := b.moved
|
|
b.moved = make(chan struct{})
|
|
b.mu.Unlock()
|
|
close(moved)
|
|
}
|
|
|
|
// holdsRevision says whether the builder's picture of branch holds the source revision and has taken in every entry of the
|
|
// ask from that branch.
|
|
func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuilds.BuildEntry) bool {
|
|
held, ok := b.Picture(branch)
|
|
if !ok || held < source {
|
|
return false
|
|
}
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
for _, e := range entries {
|
|
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
|
if err != nil {
|
|
return true // the build key is refused where the entry is built, with why
|
|
}
|
|
if b.origin[key].Branch != branch && b.heard[branch+"\x00"+e.Module].Fp2 != e.Fingerprint {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number) and that
|
|
// revision names every entry of the ask (H-A-205, H-A-222); else failed or could not run, with its words.
|
|
func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) {
|
|
below := fmt.Sprintf("revision %d of %s is not named by the builder's picture, which holds source revision %d", run.Source, run.Branch, held)
|
|
if run.Source < held {
|
|
below = fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held)
|
|
}
|
|
if b.KeptRevision == nil || run.Revision <= 0 {
|
|
return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by"
|
|
}
|
|
source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision)
|
|
if err != nil {
|
|
b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err)
|
|
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err)
|
|
}
|
|
// An ask names the revision and the source of one revision (H-A-218, H-A-219): a kept revision of another source is
|
|
// not the ask's.
|
|
if source != run.Source {
|
|
b.logf("%s: %s, and the catalog's revision %d is of source revision %d, the ask names %d: the step fails, nothing is built", run.Pipeline, below, run.Revision, source, run.Source)
|
|
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d, the ask names %d", below, run.Revision, source, run.Source)
|
|
}
|
|
for _, e := range run.Entries {
|
|
if fps[e.Module] != e.Fingerprint {
|
|
b.logf("%s: %s, and the catalog's revision %d names %s at %q, not %s: the step fails, nothing is built", run.Pipeline, below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
|
|
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
|
|
}
|
|
}
|
|
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
|
|
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
|
|
b.mu.Lock()
|
|
for _, e := range run.Entries {
|
|
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
|
if err != nil || b.origin[key].Branch != "" {
|
|
continue
|
|
}
|
|
first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module)
|
|
b.trunkFP[key] = true
|
|
b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first}
|
|
}
|
|
b.mu.Unlock()
|
|
b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision)
|
|
return "", ""
|
|
}
|
|
|
|
// awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the
|
|
// step's outcome failed, at the builder's stop could not run, each with its words.
|
|
func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) {
|
|
if b.Picture == nil || run.Source <= 0 || run.Branch == "" {
|
|
return "", ""
|
|
}
|
|
if held, ok := b.Picture(run.Branch); !b.holdsRevision(run.Branch, run.Source, run.Entries) {
|
|
below := ok && run.Source < held
|
|
if (run.Revision > 0 && b.KeptRevision != nil) || below {
|
|
// By its number (H-A-222), whatever the source the picture holds; for an ask naming none, only below the
|
|
// picture (H-A-205). A kept revision of another source or naming other fingerprints is a mismatch the wait
|
|
// cannot mend: it fails at once; only a revision the catalog does not answer is waited for.
|
|
outcome, said := b.keptRevisionHolds(ctx, run, held)
|
|
if outcome == "" || below || outcome == meshbuilds.StepEndedOutcomeFailed {
|
|
return outcome, said
|
|
}
|
|
}
|
|
}
|
|
bound := b.revisionWait()
|
|
notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch)
|
|
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
|
|
defer stop()
|
|
told := false
|
|
for {
|
|
b.mu.Lock()
|
|
moved := b.moved
|
|
b.mu.Unlock()
|
|
if b.holdsRevision(run.Branch, run.Source, run.Entries) {
|
|
if told {
|
|
b.logf("%s: revision %d of %s is in the builder's picture: the build goes on", run.Pipeline, run.Source, run.Branch)
|
|
}
|
|
return "", ""
|
|
}
|
|
if !told {
|
|
b.logf("%s: %s yet: the ask waits for it, at most %s", run.Pipeline, notHeld, bound)
|
|
told = true
|
|
}
|
|
select {
|
|
case <-moved:
|
|
case <-wait.Done():
|
|
if ctx.Err() != nil {
|
|
return meshbuilds.StepEndedOutcomeCouldNotRun, notHeld + ": the builder stopped before it was"
|
|
}
|
|
b.logf("%s: %s within %s: the step fails, nothing is built", run.Pipeline, notHeld, bound)
|
|
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notHeld, bound)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry
|
|
// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its
|
|
// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions,
|
|
// H-A-194), then waits for the registry seat's published of that package path in that series (the
|
|
// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to
|
|
// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a
|
|
// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and
|
|
// nothing is built.
|
|
|
|
// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default.
|
|
const DefaultPackageWait = 10 * time.Minute
|
|
|
|
func (b *Builder) packageWait() time.Duration {
|
|
if b.PackageWait > 0 {
|
|
return b.PackageWait
|
|
}
|
|
return DefaultPackageWait
|
|
}
|
|
|
|
// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the
|
|
// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package
|
|
// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing.
|
|
func (b *Builder) PackagePublished(path, version string) {
|
|
v := strings.TrimPrefix(version, "v")
|
|
series, ok := seriesOfVersion(v)
|
|
if path == "" || !ok {
|
|
return
|
|
}
|
|
b.init()
|
|
b.mu.Lock()
|
|
in := b.publishedPins[path]
|
|
if in == nil {
|
|
in = map[string]string{}
|
|
b.publishedPins[path] = in
|
|
}
|
|
if old := in[series]; old == "" || compareVersions(v, old) > 0 {
|
|
in[series] = v
|
|
}
|
|
b.mu.Unlock()
|
|
b.pictureMoved()
|
|
}
|
|
|
|
// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none.
|
|
func (b *Builder) publishedPin(path, series string) string {
|
|
b.mu.Lock()
|
|
v := b.publishedPins[path][series]
|
|
b.mu.Unlock()
|
|
if v == "" {
|
|
return ""
|
|
}
|
|
return "v" + v
|
|
}
|
|
|
|
// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest
|
|
// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and
|
|
// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops.
|
|
func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) {
|
|
var unpinned []string
|
|
for p, v := range packages {
|
|
if v == "" {
|
|
unpinned = append(unpinned, p)
|
|
}
|
|
}
|
|
if len(unpinned) == 0 {
|
|
return packages, "", ""
|
|
}
|
|
slices.Sort(unpinned)
|
|
if b.Manifests == nil {
|
|
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0])
|
|
}
|
|
man, err := b.Manifests.Manifest(ctx, digest)
|
|
if err != nil {
|
|
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err)
|
|
}
|
|
// What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it.
|
|
for _, p := range unpinned {
|
|
if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" {
|
|
continue
|
|
}
|
|
vs, err := b.PublishedVersions(ctx, p)
|
|
if err != nil {
|
|
b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err)
|
|
continue
|
|
}
|
|
for _, v := range vs {
|
|
b.PackagePublished(p, v)
|
|
}
|
|
}
|
|
bound := b.packageWait()
|
|
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
|
|
defer stop()
|
|
told := false
|
|
for {
|
|
b.mu.Lock()
|
|
moved := b.moved
|
|
b.mu.Unlock()
|
|
pins := map[string]string{}
|
|
missing, missingSeries := "", ""
|
|
for _, p := range unpinned {
|
|
series := man.AcceptedSeries(p)
|
|
pins[p] = b.publishedPin(p, series)
|
|
if pins[p] == "" && missing == "" {
|
|
missing, missingSeries = p, series
|
|
}
|
|
}
|
|
if missing == "" {
|
|
out := maps.Clone(packages)
|
|
maps.Copy(out, pins)
|
|
if told {
|
|
b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins)
|
|
}
|
|
return out, "", ""
|
|
}
|
|
notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries)
|
|
if !told {
|
|
b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound)
|
|
told = true
|
|
}
|
|
select {
|
|
case <-moved:
|
|
case <-wait.Done():
|
|
if ctx.Err() != nil {
|
|
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
|
|
}
|
|
b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound)
|
|
return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
|
|
}
|
|
}
|
|
}
|
|
|
|
// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form.
|
|
func seriesOfVersion(v string) (string, bool) {
|
|
if _, ok := versionParts(v); !ok {
|
|
return "", false
|
|
}
|
|
major, rest, _ := strings.Cut(v, ".")
|
|
minor, _, _ := strings.Cut(rest, ".")
|
|
return major + "." + minor, true
|
|
}
|
|
|
|
// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form.
|
|
func versionParts(v string) ([3]int, bool) {
|
|
var out [3]int
|
|
parts := strings.Split(v, ".")
|
|
if len(parts) != 3 {
|
|
return out, false
|
|
}
|
|
for i, p := range parts {
|
|
n, err := strconv.Atoi(p)
|
|
if err != nil || n < 0 {
|
|
return out, false
|
|
}
|
|
out[i] = n
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// compareVersions orders two MAJOR.MINOR.PATCH versions numerically.
|
|
func compareVersions(a, b string) int {
|
|
x, _ := versionParts(a)
|
|
y, _ := versionParts(b)
|
|
for i := range x {
|
|
if x[i] != y[i] {
|
|
return x[i] - y[i]
|
|
}
|
|
}
|
|
return 0
|
|
}
|