H-A-489: build only a branch's newest entry; withdrawn is final
Two fingerprints of one module took the series from each other (nats flipped between 0.1.140 and 0.1.152): a later look built an older fingerprint again, and its reservation took the newer one's over. The builder now refuses, before it reserves or builds, a fingerprint of the first watched branch that is no longer the newest entry heard there, saying once per key that it is superseded; a step ask of it fails so, never waits. It looks again just before it reserves. A key withdrawn is taken again by no look, only by a person's build again. Builds: design 47 H2 H-A-489
This commit is contained in:
@@ -170,6 +170,7 @@ func Unreferenced(b builds.BuilderBuild) bool {
|
||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
|
||||
// - a failed key withdrawn (withdrawnPrefix) is final for its fingerprint, taken again only by a person's build again (H-A-489);
|
||||
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
|
||||
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
|
||||
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
|
||||
@@ -229,6 +230,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, withdrawnPrefix) && !opt.ByHand:
|
||||
return nil, nil // withdrawn is final for its fingerprint: no look builds it again (H-A-489)
|
||||
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent:
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
|
||||
|
||||
@@ -177,6 +177,8 @@ type Builder struct {
|
||||
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
|
||||
// a package with no pin waits for one (H-A-192).
|
||||
publishedPins map[string]map[string]string
|
||||
// supersededSaid are the build keys refused as superseded (H-A-489), so each is said once.
|
||||
supersededSaid sync.Map
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -310,11 +312,49 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
if superseded != nil {
|
||||
// A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn
|
||||
// from the build seats (ADR 0664 §4).
|
||||
superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2})
|
||||
superseded(buildrun.Withdrawn{Why: supersededWhy(key, old.Fp2, m.Fp2)})
|
||||
}
|
||||
b.consider(ctx, key, m)
|
||||
}
|
||||
|
||||
// supersededWhy is the words of a build withdrawn because a newer fingerprint of its module superseded it.
|
||||
func supersededWhy(module, fp, newer string) string {
|
||||
return "the build of " + module + " at " + fp + " was superseded by the fingerprint " + newer
|
||||
}
|
||||
|
||||
// withdrawnPrefix opens the error of a key whose build was withdrawn (H-A-225, ADR 0664 §4). It is final for its fingerprint
|
||||
// (H-A-489): no look builds it again; a person's build again does.
|
||||
const withdrawnPrefix = "withdrawn: "
|
||||
|
||||
// supersededBy is the fingerprint that supersedes fp, a fingerprint of module whose build key is key, when fp is of its
|
||||
// registration's first watched branch and is no longer the newest entry heard on that branch (H-A-489); "" otherwise. A key no
|
||||
// entry named is of the first branch, as originOf has it.
|
||||
func (b *Builder) supersededBy(module, key, fp string) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
o, ok := b.origin[key]
|
||||
if !ok {
|
||||
o = Origin{First: true}
|
||||
}
|
||||
if !o.First {
|
||||
return ""
|
||||
}
|
||||
newest, heard := b.heard[o.Branch+"\x00"+module]
|
||||
if !heard || newest.Fp2 == "" || newest.Fp2 == fp {
|
||||
return ""
|
||||
}
|
||||
return newest.Fp2
|
||||
}
|
||||
|
||||
// saySuperseded says, once per key, that the build of fp is refused as superseded by newer (H-A-489), and gives the words.
|
||||
func (b *Builder) saySuperseded(module, key, fp, newer string) string {
|
||||
why := "is not built: superseded by " + newer + ", the newest entry heard on its first branch"
|
||||
if _, said := b.supersededSaid.LoadOrStore(key, true); !said {
|
||||
b.logf("%s at %s: %s", module, fp, why)
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
|
||||
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
|
||||
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
|
||||
@@ -637,6 +677,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
whyEnded = fmt.Sprintf(format, args...)
|
||||
b.logf("%s at %s: %s", module, m.Fp2, whyEnded)
|
||||
}
|
||||
// A fingerprint no longer the newest entry heard on its first branch is neither reserved for nor built (H-A-489): two
|
||||
// fingerprints of one module never take the series from each other.
|
||||
if newer := b.supersededBy(module, key, m.Fp2); newer != "" {
|
||||
whyEnded = b.saySuperseded(module, key, m.Fp2, newer)
|
||||
return
|
||||
}
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
@@ -727,7 +773,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
}
|
||||
var reserved versions.ReservedVersion
|
||||
hasReserved := false
|
||||
hasReserved, supersededNow := false, ""
|
||||
if rebuild && cl.Build.State == builds.BuildBuilding {
|
||||
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
|
||||
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
|
||||
@@ -737,10 +783,16 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
reserving = true
|
||||
} else if reserving && cl.Build.State != builds.BuildDone {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
// Looked at again just before it reserves (H-A-489): a newer entry heard since the claim is not taken over.
|
||||
if supersededNow = b.supersededBy(module, key, m.Fp2); supersededNow == "" {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
}
|
||||
}
|
||||
var result builds.BuilderBuild
|
||||
if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
if supersededNow != "" {
|
||||
// Ended as the newer entry's take-in withdraws a build in flight: final for this fingerprint.
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + supersededWhy(module, m.Fp2, supersededNow)})}
|
||||
} else if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
// A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would
|
||||
// stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again.
|
||||
result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be")
|
||||
@@ -765,7 +817,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var w buildrun.Withdrawn
|
||||
if ctx.Err() == nil && errors.As(context.Cause(run), &w) {
|
||||
// Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them).
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})}
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + w.Why})}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
|
||||
@@ -333,8 +333,9 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
asked, release := make(chan struct{}), make(chan struct{})
|
||||
answer := r.answer
|
||||
var once sync.Once
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
close(asked)
|
||||
once.Do(func() { close(asked) })
|
||||
<-release
|
||||
return answer(a)
|
||||
}
|
||||
@@ -356,6 +357,62 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
|
||||
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
|
||||
}
|
||||
// H-A-489: withdrawn is final for the fingerprint: a later look builds it not again; a person's build again does.
|
||||
n, rev := len(r.asks.Asks()), r.st.revs[r.key]
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); len(r.asks.Asks()) != n || r.st.revs[r.key] != rev || !strings.HasPrefix(v.Error, "withdrawn: ") {
|
||||
t.Errorf("a later look built the withdrawn key again: %d asks (was %d), the key %+v", len(r.asks.Asks()), n, v)
|
||||
}
|
||||
if cl, err := r.b.Claimer.Claim(context.Background(), "m", fp1, mdigest('a'), ClaimOptions{ByHand: true}); err != nil || cl == nil {
|
||||
t.Errorf("a person's build again of the withdrawn key was not claimed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and
|
||||
// said once; the again loop that looks at it later builds nothing.
|
||||
func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
r.b.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
|
||||
older := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'),
|
||||
Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
delete(r.bm.fails, "ace/build_go_build")
|
||||
newer := older
|
||||
newer.Fp2 = fp2
|
||||
data, err := meshcatalogclient.EncodeModule(newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
rev := r.st.revs[r.key]
|
||||
for range 2 {
|
||||
r.b.consider(context.Background(), "m", older) // as consider's again loop or a step's look takes it
|
||||
r.b.Wait()
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || r.st.revs[r.key] != rev {
|
||||
t.Errorf("the older key = %+v at revision %d (was %d), want it left as it failed", v, r.st.revs[r.key], rev)
|
||||
}
|
||||
k2, _ := builds.BuilderBuildKey("m", fp2)
|
||||
if v := valueOf(t, r.st, k2); v.State != builds.BuildDone {
|
||||
t.Errorf("the newer key = %+v, want done", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range said {
|
||||
if strings.Contains(l, fp1+": is not built: superseded by "+fp2) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("superseded said %d times, want once: %q", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||
|
||||
@@ -142,6 +142,10 @@ func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.
|
||||
}
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
|
||||
}
|
||||
if newer := b.supersededBy(entry.Module, key, entry.Fp); newer != "" {
|
||||
// Refused, not waited on (H-A-489): no look builds a superseded fingerprint, so its end would never come.
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s %s", entry.Module, entry.Fp, b.saySuperseded(entry.Module, key, entry.Fp, newer)), "refused")
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
@@ -232,7 +236,7 @@ func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed str
|
||||
switch {
|
||||
case v.State == builds.BuildDone && !Unpublished(v, b.Seats):
|
||||
return true, "", true
|
||||
case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error):
|
||||
case v.State == builds.BuildFailed && (builds.FailedUntilChanged(v.Error) || strings.HasPrefix(v.Error, withdrawnPrefix)):
|
||||
return false, key + ": " + v.Error, true
|
||||
}
|
||||
return false, "", false
|
||||
|
||||
@@ -460,6 +460,24 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
|
||||
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
|
||||
}
|
||||
// H-A-489: once the picture heard the newer entry of m on the first branch, the older ask is refused superseded and
|
||||
// nothing is built or reserved for it; the newer entry's build keeps the series.
|
||||
rn, resn, entryn := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
rn.b.RevisionWait = time.Nanosecond
|
||||
rn.b.FirstBranch = r.b.FirstBranch
|
||||
rn.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return 45, map[string]string{"m": fp1}, nil
|
||||
}
|
||||
entryn.Fp = fp2
|
||||
rn.b.Entry(context.Background(), Origin{Branch: "main", Source: 46, First: true}, entryn)
|
||||
rn.b.Wait()
|
||||
if endn := rn.b.BuildStep(t.Context(), "a1", run); endn.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(endn.Said, "superseded by "+fp2) {
|
||||
t.Errorf("the older ask after the newer entry: end = %+v; want failed, superseded by %s", endn, fp2)
|
||||
}
|
||||
rn.b.Wait()
|
||||
if _, built := rn.st.vals[rn.key]; built || len(resn.sources) != 1 || resn.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("built %v, reserved %+v; want nothing built for the older, one reservation, the newer's", built, resn.sources)
|
||||
}
|
||||
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r2.b.RevisionWait = time.Nanosecond
|
||||
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
|
||||
Reference in New Issue
Block a user