Runner and builder: a run's build step ask names no source; a held revision wakes the builder's wait (H-A-162)
A run on request copied its pipeline's source onto its build step ask, so the builder waited for the run's candidate fingerprints in its picture of the branch, where they never come, and failed every run past the bound. The seat mesh-builds' contract leaves a run's source absent: the runner now does. A revision the builder's view takes in whole that changes no entry now also wakes a waiting ask, told by the catalog client's new Held. Builds: design 47 H H-A-162, H-A-155
This commit is contained in:
@@ -35,6 +35,7 @@ import (
|
|||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"
|
sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/operatorask"
|
"git.novox.be/novox/mesh-sdk/go/operatorask"
|
||||||
"git.novox.be/novox/mesh-sdk/go/process"
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
@@ -458,6 +459,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// a holder of the seat that is down holds no other part of it).
|
// a holder of the seat that is down holds no other part of it).
|
||||||
half.b = b
|
half.b = b
|
||||||
b.Picture = half.picture
|
b.Picture = half.picture
|
||||||
|
branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162)
|
||||||
half.ready = func() {
|
half.ready = func() {
|
||||||
if err := b.Sweep(ctx, half.pictured()...); err != nil {
|
if err := b.Sweep(ctx, half.pictured()...); err != nil {
|
||||||
errs <- err
|
errs <- err
|
||||||
|
|||||||
@@ -12,7 +12,8 @@ import (
|
|||||||
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
|
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
|
||||||
// ask names its branch and source revision, and the builder reserves the version before it builds. When the builder's picture
|
// ask names its branch and source revision, and the builder reserves the version before it builds. When the builder's picture
|
||||||
// of that branch does not hold that source revision yet, or holds it but has not taken in the entry of the asked fingerprint
|
// of that branch does not hold that source revision yet, or holds it but has not taken in the entry of the asked fingerprint
|
||||||
// from it, the ask waits for the picture to take it in, woken by each entry the picture takes in (take, EntryGone), never a
|
// from it, the ask waits for the picture to take it in, woken by each entry the picture takes in (take, EntryGone) and each
|
||||||
|
// revision its view holds (PictureHeld, H-A-162), never a
|
||||||
// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the
|
// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the
|
||||||
// branch, and nothing is built. A step ask with no source (a run on request) does not wait.
|
// branch, and nothing is built. A step ask with no source (a run on request) does not wait.
|
||||||
|
|
||||||
@@ -26,7 +27,14 @@ func (b *Builder) revisionWait() time.Duration {
|
|||||||
return DefaultRevisionWait
|
return DefaultRevisionWait
|
||||||
}
|
}
|
||||||
|
|
||||||
// pictureMoved wakes every step ask waiting for the picture: an entry was taken in or went.
|
// PictureHeld is told each revision a branch's view takes in whole, whether or not an entry changed (H-A-162): every step
|
||||||
|
// ask waiting for its revision looks again.
|
||||||
|
func (b *Builder) PictureHeld() {
|
||||||
|
b.init()
|
||||||
|
b.pictureMoved()
|
||||||
|
}
|
||||||
|
|
||||||
|
// pictureMoved wakes every step ask waiting for the picture: an entry was taken in or went, or a revision was held.
|
||||||
func (b *Builder) pictureMoved() {
|
func (b *Builder) pictureMoved() {
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
moved := b.moved
|
moved := b.moved
|
||||||
|
|||||||
@@ -115,6 +115,38 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
|
|||||||
}
|
}
|
||||||
t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision)
|
t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision)
|
||||||
t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails)
|
t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails)
|
||||||
|
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
|
||||||
|
}
|
||||||
|
|
||||||
|
func aRevisionHeldWakesTheAsk(t *testing.T) {
|
||||||
|
var held atomic.Int64
|
||||||
|
held.Store(40)
|
||||||
|
asked := make(chan struct{}, 1)
|
||||||
|
r, res, entry := revisionRig(t, &held, asked)
|
||||||
|
r.b.Entry(context.Background(), Origin{Branch: "main", Source: 40, First: true}, entry) // the entry, unchanged since 40
|
||||||
|
r.b.Wait()
|
||||||
|
ended := make(chan meshbuilds.StepEnded, 1)
|
||||||
|
go func() {
|
||||||
|
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
|
||||||
|
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-asked:
|
||||||
|
case end := <-ended:
|
||||||
|
t.Fatalf("the ask ended before the picture held its revision: %+v", end)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the ask never read the builder's picture")
|
||||||
|
}
|
||||||
|
held.Store(41)
|
||||||
|
r.b.PictureHeld() // revision 8, from source 41, changes no entry
|
||||||
|
select {
|
||||||
|
case end := <-ended:
|
||||||
|
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 {
|
||||||
|
t.Errorf("end = %+v, reserved from %+v; want passed with its version reserved", end, res.sources)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("a revision held did not wake the ask")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// revisionRig is a rig whose builder reserves, its step entry m at fp1, its picture of main holding the source revision held.
|
// revisionRig is a rig whose builder reserves, its step entry m at fp1, its picture of main holding the source revision held.
|
||||||
|
|||||||
@@ -74,6 +74,8 @@ type Branches struct {
|
|||||||
holders atomic.Pointer[seatHolders]
|
holders atomic.Pointer[seatHolders]
|
||||||
// Changed, when not nil, is told of each change to a branch's view (module, its entry, deleted), set before Wire.
|
// Changed, when not nil, is told of each change to a branch's view (module, its entry, deleted), set before Wire.
|
||||||
Changed func(branch, module string, entry json.RawMessage, deleted bool, at time.Time)
|
Changed func(branch, module string, entry json.RawMessage, deleted bool, at time.Time)
|
||||||
|
// Held, when not nil, is told of each revision a branch's view takes in whole (Catalog.Held; H-A-162), set before Wire.
|
||||||
|
Held func(branch string, revision seat.CatalogRevision)
|
||||||
// Adopted, when not nil, is told on a goroutine of its own of each branch whose view was adopted, with that view (a part that waits for
|
// Adopted, when not nil, is told on a goroutine of its own of each branch whose view was adopted, with that view (a part that waits for
|
||||||
// its first branch's first view, design 47 G §507), set before Wire.
|
// its first branch's first view, design 47 G §507), set before Wire.
|
||||||
Adopted func(branch string, c *Catalog)
|
Adopted func(branch string, c *Catalog)
|
||||||
@@ -253,6 +255,11 @@ func (b *Branches) reconcile() {
|
|||||||
b.Changed(branch, module, entry, deleted, at)
|
b.Changed(branch, module, entry, deleted, at)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
c.Held = func(r seat.CatalogRevision) {
|
||||||
|
if b.Held != nil {
|
||||||
|
b.Held(branch, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
// A catalog-revision also asks again each entry whose ask failed in transport (design 47 H, H-A-60).
|
// A catalog-revision also asks again each entry whose ask failed in transport (design 47 H, H-A-60).
|
||||||
hear := func(ctx context.Context, data []byte) error {
|
hear := func(ctx context.Context, data []byte) error {
|
||||||
err := c.Hear(ctx, data)
|
err := c.Hear(ctx, data)
|
||||||
|
|||||||
@@ -59,6 +59,9 @@ type Catalog struct {
|
|||||||
waits int64
|
waits int64
|
||||||
// made is when the view's numbering last started again (H-A-141), the follower's Created; zero is catalogCreated.
|
// made is when the view's numbering last started again (H-A-141), the follower's Created; zero is catalogCreated.
|
||||||
made time.Time
|
made time.Time
|
||||||
|
// Held, when not nil, is told each revision the view takes in whole, once it holds it, whether or not an entry
|
||||||
|
// changed (design 47 H H-A-162): a part waiting for a revision wakes on it. Set before Start.
|
||||||
|
Held func(seat.CatalogRevision)
|
||||||
}
|
}
|
||||||
|
|
||||||
// created is madeLocked, taking mu.
|
// created is madeLocked, taking mu.
|
||||||
@@ -232,7 +235,9 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) {
|
|||||||
}
|
}
|
||||||
rev := a.Revision.Revision
|
rev := a.Revision.Revision
|
||||||
holds := false
|
holds := false
|
||||||
if a.Next == "" {
|
whole := a.Next == ""
|
||||||
|
held := *a.Revision
|
||||||
|
if whole {
|
||||||
c.held, c.paging = c.paging, nil
|
c.held, c.paging = c.paging, nil
|
||||||
if c.waits <= rev {
|
if c.waits <= rev {
|
||||||
c.waits, holds = 0, true
|
c.waits, holds = 0, true
|
||||||
@@ -242,6 +247,9 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) {
|
|||||||
if holds {
|
if holds {
|
||||||
c.Follower.Holds(ViewName(rev))
|
c.Follower.Holds(ViewName(rev))
|
||||||
}
|
}
|
||||||
|
if whole && c.Held != nil {
|
||||||
|
c.Held(held)
|
||||||
|
}
|
||||||
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: c.created(), Values: values, Next: a.Next, At: time.Now()}, nil
|
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: c.created(), Values: values, Next: a.Next, At: time.Now()}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -83,6 +84,14 @@ func TestCatalogIsAViewPerWatchedBranch(t *testing.T) {
|
|||||||
c.Follower.Wait = time.Nanosecond
|
c.Follower.Wait = time.Nanosecond
|
||||||
c.Follower.Logf = logf
|
c.Follower.Logf = logf
|
||||||
}
|
}
|
||||||
|
// H-A-162: each revision the view takes in whole is told to Held.
|
||||||
|
var heldMu sync.Mutex
|
||||||
|
var held []int64
|
||||||
|
main.Held = func(r seat.CatalogRevision) {
|
||||||
|
heldMu.Lock()
|
||||||
|
defer heldMu.Unlock()
|
||||||
|
held = append(held, r.Revision)
|
||||||
|
}
|
||||||
if main.Follower.State == rel.Follower.State || main.Follower.State != "catalog-revisions.main" || rel.Follower.State != views.BranchState(seat.StateCatalogRevisions, "release/1") {
|
if main.Follower.State == rel.Follower.State || main.Follower.State != "catalog-revisions.main" || rel.Follower.State != views.BranchState(seat.StateCatalogRevisions, "release/1") {
|
||||||
t.Fatalf("views are named %q and %q", main.Follower.State, rel.Follower.State)
|
t.Fatalf("views are named %q and %q", main.Follower.State, rel.Follower.State)
|
||||||
}
|
}
|
||||||
@@ -116,17 +125,25 @@ func TestCatalogIsAViewPerWatchedBranch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
for {
|
for {
|
||||||
if h, _ := main.View(); h.Revision.Revision == 4 {
|
heldMu.Lock()
|
||||||
|
told := len(held)
|
||||||
|
heldMu.Unlock()
|
||||||
|
if h, _ := main.View(); h.Revision.Revision == 4 && told == 2 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if time.Now().After(deadline) {
|
if time.Now().After(deadline) {
|
||||||
t.Fatal("main never held revision 4")
|
t.Fatal("main never held revision 4, told to Held")
|
||||||
}
|
}
|
||||||
time.Sleep(time.Millisecond)
|
time.Sleep(time.Millisecond)
|
||||||
}
|
}
|
||||||
if n := f.asked("release/1"); n != 1 {
|
if n := f.asked("release/1"); n != 1 {
|
||||||
t.Errorf("release/1 asked %d times after main's revision", n)
|
t.Errorf("release/1 asked %d times after main's revision", n)
|
||||||
}
|
}
|
||||||
|
heldMu.Lock()
|
||||||
|
if !slices.Equal(held, []int64{3, 4}) {
|
||||||
|
t.Errorf("Held was told %v, want 3 then 4", held)
|
||||||
|
}
|
||||||
|
heldMu.Unlock()
|
||||||
logMu.Lock()
|
logMu.Lock()
|
||||||
said := false
|
said := false
|
||||||
for _, l := range logged {
|
for _, l := range logged {
|
||||||
|
|||||||
@@ -98,8 +98,14 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
|
|||||||
build := seat.StepResult{Step: seat.StepResultStepBuild, Module: m.Module, Status: seat.StepResultStatusRequired, Fingerprint: m.Fingerprint,
|
build := seat.StepResult{Step: seat.StepResultStepBuild, Module: m.Module, Status: seat.StepResultStatusRequired, Fingerprint: m.Fingerprint,
|
||||||
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(startOf(p, now), p.Pipeline, m.Module, "", seat.StepResultStepBuild)}
|
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(startOf(p, now), p.Pipeline, m.Module, "", seat.StepResultStepBuild)}
|
||||||
steps = append(steps, build)
|
steps = append(steps, build)
|
||||||
|
// A run on request's build step ask names no source (the seat mesh-builds' contract; H-A-162): its candidate's
|
||||||
|
// fingerprints are never in the builder's picture of the branch, so it waits for no revision there.
|
||||||
|
source := p.Source
|
||||||
|
if p.Run != "" {
|
||||||
|
source = 0
|
||||||
|
}
|
||||||
asks = append(asks, Asked{Seat: builds.Seat, ID: build.Ask, Commit: m.Commit, Ask: builds.StepRun{Pipeline: p.Pipeline, Step: builds.StepRunStepBuild, Branch: p.Branch,
|
asks = append(asks, Asked{Seat: builds.Seat, ID: build.Ask, Commit: m.Commit, Ask: builds.StepRun{Pipeline: p.Pipeline, Step: builds.StepRunStepBuild, Branch: p.Branch,
|
||||||
Revision: p.Revision, Source: p.Source, Entries: []builds.BuildEntry{{Module: m.Module, Commit: m.Commit, Fingerprint: m.Fingerprint}}}})
|
Revision: p.Revision, Source: source, Entries: []builds.BuildEntry{{Module: m.Module, Commit: m.Commit, Fingerprint: m.Fingerprint}}}})
|
||||||
for _, a := range arts {
|
for _, a := range arts {
|
||||||
name := a.Ref
|
name := a.Ref
|
||||||
if _, after, ok := strings.Cut(a.Ref, ":"); ok {
|
if _, after, ok := strings.Cut(a.Ref, ":"); ok {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||||
)
|
)
|
||||||
@@ -97,6 +98,19 @@ func TestRunOnRequest(t *testing.T) {
|
|||||||
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
|
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
|
||||||
t.Errorf("a placed run is still unplaced: %v", ids)
|
t.Errorf("a placed run is still unplaced: %v", ids)
|
||||||
}
|
}
|
||||||
|
// H-A-162: a run's build step asks name no source, as the seat mesh-builds' contract says of a run on request.
|
||||||
|
builtAsks := 0
|
||||||
|
for _, x := range asks.asks {
|
||||||
|
if r, ok := x.(builds.StepRun); ok {
|
||||||
|
builtAsks++
|
||||||
|
if r.Source != 0 {
|
||||||
|
t.Errorf("the run's build step ask names source %d, want none", r.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if builtAsks == 0 {
|
||||||
|
t.Errorf("the run asked no build step: %v", asks.asks)
|
||||||
|
}
|
||||||
if n, _ := s.PlaceRun(ev, Half{Revision: 12, Entries: cand}, base); n != 0 {
|
if n, _ := s.PlaceRun(ev, Half{Revision: 12, Entries: cand}, base); n != 0 {
|
||||||
t.Errorf("the revision heard again placed %d", n)
|
t.Errorf("the revision heard again placed %d", n)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user