diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go new file mode 100644 index 00000000..e67548a8 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -0,0 +1,94 @@ +package build + +import ( + "context" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" +) + +// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step +// ask names its branch and source revision, and the builder reserves the version before it builds. When the builder's picture +// of that branch does not hold that source revision yet, or holds it but has not taken in the entry of the asked fingerprint +// from it, the ask waits for the picture to take it in, woken by each entry the picture takes in (take, EntryGone), never a +// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the +// branch, and nothing is built. A step ask with no source (a run on request) does not wait. + +// DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156). +const DefaultRevisionWait = 2 * time.Minute + +func (b *Builder) revisionWait() time.Duration { + if b.RevisionWait > 0 { + return b.RevisionWait + } + return DefaultRevisionWait +} + +// pictureMoved wakes every step ask waiting for the picture: an entry was taken in or went. +func (b *Builder) pictureMoved() { + b.mu.Lock() + moved := b.moved + b.moved = make(chan struct{}) + b.mu.Unlock() + close(moved) +} + +// holdsRevision says whether the builder's picture of branch holds the source revision and has taken in every entry of the +// ask from that branch. +func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuilds.BuildEntry) bool { + held, ok := b.Picture(branch) + if !ok || held < source { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + for _, e := range entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil { + return true // the build key is refused where the entry is built, with why + } + if b.origin[key].Branch != branch && b.heard[branch+"\x00"+e.Module].Fp2 != e.Fingerprint { + return false + } + } + return true +} + +// awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the +// step's outcome failed, at the builder's stop could not run, each with its words. +func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) { + if b.Picture == nil || run.Source <= 0 || run.Branch == "" { + return "", "" + } + bound := b.revisionWait() + notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch) + wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8) + defer stop() + told := false + for { + b.mu.Lock() + moved := b.moved + b.mu.Unlock() + if b.holdsRevision(run.Branch, run.Source, run.Entries) { + if told { + b.logf("%s: revision %d of %s is in the builder's picture: the build goes on", run.Pipeline, run.Source, run.Branch) + } + return "", "" + } + if !told { + b.logf("%s: %s yet: the ask waits for it, at most %s", run.Pipeline, notHeld, bound) + told = true + } + select { + case <-moved: + case <-wait.Done(): + if ctx.Err() != nil { + return meshbuilds.StepEndedOutcomeCouldNotRun, notHeld + ": the builder stopped before it was" + } + b.logf("%s: %s within %s: the step fails, nothing is built", run.Pipeline, notHeld, bound) + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notHeld, bound) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index fcabe73a..242707b5 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -107,6 +107,12 @@ type Builder struct { Numbers Numbers // Reserved reserves the version of a trunk build at its start (design 47 G §369); nil reserves nothing. Reserved Reservations + // Picture answers the source revision the builder's picture of a watched branch holds, false while it holds none (design 47 + // H H-A-155, H-A-141): a step ask naming a newer one waits for it. Nil waits for nothing. + Picture func(branch string) (source int64, held bool) + // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait + // when 0. + RevisionWait time.Duration // Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine // answered when last asked, and counted, every node the assignments names, present or away, a node with no statement // stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in @@ -149,6 +155,8 @@ type Builder struct { // builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of // the module takes its older one's build out, and the asker drops that build's asks. builds map[string]context.CancelCauseFunc + // moved is closed and made again at each entry the picture takes in or drops: the step asks waiting for a revision wake. + moved chan struct{} } type planned struct { @@ -178,6 +186,7 @@ func (b *Builder) init() { b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands, b.events = map[string]*handWatch{}, map[string]bool{} b.builds = map[string]context.CancelCauseFunc{} + b.moved = make(chan struct{}) }) } @@ -230,6 +239,7 @@ func (b *Builder) EntryGone(branch string, first bool, module string) { b.mu.Lock() delete(b.heard, branch+"\x00"+module) b.mu.Unlock() + b.pictureMoved() if first { b.Forget(module) } @@ -256,6 +266,7 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog. b.edges[bk] = e } b.mu.Unlock() + b.pictureMoved() if superseded != nil { // A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn // from the build seats (ADR 0664 §4). diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index ebfa5729..eb40ed3a 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -34,6 +34,10 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui if b.StepEntry == nil { return finish(meshbuilds.StepEndedOutcomeCouldNotRun, "the builder reads no entry of the seat mesh-catalog") } + // The version is reserved before the build (H-A-155): the ask waits for the builder's picture to hold its revision. + if outcome, said := b.awaitRevision(ctx, run); outcome != "" { + return finish(outcome, said) + } var failed []string for _, e := range run.Entries { outcome, said := b.buildEntry(ctx, e) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 0cd9c551..7dbbeb1c 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "strings" + "sync/atomic" "testing" "time" @@ -112,6 +113,79 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) } }) } + t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision) + t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails) +} + +// revisionRig is a rig whose builder reserves, its step entry m at fp1, its picture of main holding the source revision held. +func revisionRig(t *testing.T, held *atomic.Int64, asked chan struct{}) (*rig, *fakeReservations, meshcatalog.CatalogEntry) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{filled: true} + r.b.Reserved = res + entry := meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog", Path: "modules/m", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1, + ManifestDigest: mdigest('a'), Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)}} + r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil } + r.b.Picture = func(branch string) (int64, bool) { + select { + case asked <- struct{}{}: + default: + } + return held.Load(), branch == "main" && held.Load() > 0 + } + return r, res, entry +} + +func aStepAskWaitsForItsRevision(t *testing.T) { + var held atomic.Int64 + held.Store(40) + asked := make(chan struct{}, 1) + r, res, entry := revisionRig(t, &held, asked) + ended := make(chan meshbuilds.StepEnded, 1) + go func() { + ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + }() + select { + case <-asked: // the ask read the picture, which holds source revision 40 + case end := <-ended: + t.Fatalf("the ask ended before the picture held its revision: %+v, reserved from %+v", end, res.sources) + case <-time.After(5 * time.Second): + t.Fatal("the ask never read the builder's picture") + } + if _, ok := r.st.vals[r.key]; ok { + t.Fatal("the ask built before the picture held its revision") + } + held.Store(41) + r.b.Entry(context.Background(), Origin{Branch: "main", Source: 41, First: true}, entry) // the picture takes revision 41 in + var end meshbuilds.StepEnded + select { + case end = <-ended: + case <-time.After(5 * time.Second): + t.Fatal("the ask was not woken by the entry the picture took in") + } + r.b.Wait() + if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 { + t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources) + } +} + +func aStepAskPastItsRevisionBoundFails(t *testing.T) { + var held atomic.Int64 + held.Store(40) + r, res, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r.b.RevisionWait = time.Nanosecond + end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + r.b.Wait() + if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "revision 41 of main not in the builder's picture") || built || len(res.sources) != 0 { + t.Errorf("end = %+v, built %v, reserved %+v; want failed naming revision 41 of main, nothing built", end, built, res.sources) + } + // A step ask with no source (a run on request) does not wait (H-A-156). + end = r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + if strings.Contains(end.Said, "not in the builder's picture") { + t.Errorf("an ask with no source waited: %+v", end) + } } // digestOf is the manifest digest a step's entry names: the rig's manifest, or one no store holds, whose plan cannot be read.