H-A-492: record a version's digests under the builder's own bound
A takeover or an end cancelling a build's context cancelled the record of its reservation's digests eight times in a day, each leaving a passed build unreleased. The record now runs under the builder's own bound (one minute, not cancelled with the build), and a done build whose digests are not recorded is recorded from its done key at the builder's next look and at its start (before H-A-484's start check holds it against the released versions), said each time. Builds: design 47 H2 H-A-492
This commit is contained in:
@@ -259,6 +259,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492)
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
|
||||
@@ -57,6 +57,9 @@ type reservations struct {
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492);
|
||||
// nil records none.
|
||||
recordAgain func(ctx context.Context, module, fp, when string) (string, error)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands
|
||||
@@ -363,12 +366,20 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea
|
||||
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
|
||||
// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended.
|
||||
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
|
||||
// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key
|
||||
// (design 47 H2 H-A-492), so it is held against the released versions too.
|
||||
func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
if rv.recordedAtStart(ctx, all) {
|
||||
if all, err = rv.r.All(ctx); err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
read := map[string][]string{}
|
||||
for _, v := range builtOf(all) {
|
||||
if versions.NotStanding(v, true) != "" {
|
||||
@@ -392,6 +403,26 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47
|
||||
// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said.
|
||||
func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool {
|
||||
if rv.recordAgain == nil {
|
||||
return false
|
||||
}
|
||||
recorded := false
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start")
|
||||
if err != nil {
|
||||
rv.logf("%v", err)
|
||||
}
|
||||
recorded = recorded || version != ""
|
||||
}
|
||||
return recorded
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
|
||||
@@ -219,6 +219,23 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
// stands.
|
||||
refuse = ""
|
||||
tags = nil
|
||||
// Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them
|
||||
// recorded (the builder's RecordAgain), and is then held against the released versions as a built one is.
|
||||
if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var recordedAt []string
|
||||
rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) {
|
||||
recordedAt = append(recordedAt, module+" "+fp+" "+when)
|
||||
_, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest
|
||||
return "0.1.0", err
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) {
|
||||
t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt)
|
||||
}
|
||||
catalog, rv.recordAgain = nil, nil
|
||||
var ended []string
|
||||
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
|
||||
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
|
||||
|
||||
@@ -122,6 +122,53 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return out
|
||||
}
|
||||
|
||||
// recordBound bounds the record of a reservation's digests (design 47 H2 H-A-492): the builder's own bound, never the build's
|
||||
// context.
|
||||
const recordBound = time.Minute
|
||||
|
||||
// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by
|
||||
// recordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record.
|
||||
func recordContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), recordBound)
|
||||
}
|
||||
|
||||
// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47
|
||||
// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version
|
||||
// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already.
|
||||
func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) {
|
||||
b.init()
|
||||
if b.Reserved == nil {
|
||||
return "", nil
|
||||
}
|
||||
digests := b.doneDigests(ctx, module, fp)
|
||||
if digests == nil {
|
||||
return "", nil
|
||||
}
|
||||
res, ok, err := b.Reserved.ReservedFor(ctx, module, fp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err)
|
||||
}
|
||||
if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" {
|
||||
return "", nil
|
||||
}
|
||||
refs := map[string]string{}
|
||||
for ref := range res.Artifacts {
|
||||
_, artifact, _ := strings.Cut(ref, ":")
|
||||
d, have := digests[artifact]
|
||||
if !have {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when)
|
||||
}
|
||||
refs[ref] = d
|
||||
}
|
||||
rec, stop := recordContext(ctx)
|
||||
defer stop()
|
||||
if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err)
|
||||
}
|
||||
b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when)
|
||||
return res.Version, nil
|
||||
}
|
||||
|
||||
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
|
||||
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
|
||||
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
|
||||
|
||||
@@ -32,6 +32,8 @@ type fakeReservations struct {
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
|
||||
held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492)
|
||||
lost error // the record of the digests fails, as a store refusing it would
|
||||
}
|
||||
|
||||
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
@@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e
|
||||
func (f *fakeReservations) Filled() bool { return f.filled }
|
||||
|
||||
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
if v, ok := f.held[fp]; ok {
|
||||
return v, true, nil
|
||||
}
|
||||
for _, s := range f.sources {
|
||||
if s.Fingerprint == fp {
|
||||
return versions.ReservedVersion{Version: "held"}, true, nil
|
||||
@@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art
|
||||
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err // the state's write runs under ctx: a cancelled one records nothing
|
||||
}
|
||||
if f.lost != nil {
|
||||
return f.lost
|
||||
}
|
||||
f.deps = deps
|
||||
f.built = map[string]string{version: ""}
|
||||
for ref, d := range digests {
|
||||
@@ -153,6 +164,59 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a
|
||||
// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose
|
||||
// digests are not recorded is recorded at the builder's next look, said.
|
||||
func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) {
|
||||
manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
t.Run("a build's context cancelled at its end still records", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written
|
||||
r.b.Module(ctx, "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
||||
t.Fatalf("build = %+v, want done", v)
|
||||
}
|
||||
if res.built["m:tool"] != digest {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built)
|
||||
}
|
||||
})
|
||||
t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true, lost: errors.New("context canceled")}
|
||||
r.b.Reserved = res
|
||||
var journal []string
|
||||
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil {
|
||||
t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built)
|
||||
}
|
||||
res.lost = nil
|
||||
res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal)
|
||||
}
|
||||
// Recorded once: a reservation whose build ended recorded is left.
|
||||
res.built = nil
|
||||
res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built != nil {
|
||||
t.Errorf("a recorded reservation was recorded again: %v", res.built)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through
|
||||
// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why.
|
||||
func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
|
||||
|
||||
@@ -726,6 +726,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving {
|
||||
// A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492).
|
||||
if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil {
|
||||
b.logf("%v", err)
|
||||
}
|
||||
}
|
||||
if reserving && b.waitFill(module, m) {
|
||||
end("is not built yet: wait-held, the catalog has not said versions-filled")
|
||||
return
|
||||
@@ -859,8 +865,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
for _, a := range result.Artifacts {
|
||||
digests[module+":"+a.Name] = a.Digest
|
||||
}
|
||||
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
|
||||
// Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed
|
||||
// build unrecorded, so unreleased (design 47 H2 H-A-492).
|
||||
rec, stop := recordContext(ctx)
|
||||
err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key))
|
||||
stop()
|
||||
if err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err)
|
||||
}
|
||||
}
|
||||
if result.State == builds.BuildFailed {
|
||||
|
||||
Reference in New Issue
Block a user