Give a kept mover's asks ids of its kept revision and place it from that revision (H-A-286)
Review round 1: a kept mover's ask ids still came from the pipeline's start time, so a pin-held ask or a composition took the id of an ask the runner had just dropped for the old fingerprint, and a restart never matched the steps placed again; and a kept mover placed later read the branch's held view, so a lagging view gave the new fingerprint the old pins. Every ask id of a kept mover's steps (build, tests, validate, pin-held asks, composition, rerun, restart) now mixes its kept revision into the id's sum, and every placing of a kept mover (at once, later, at a restart, a rerun) reads its entries at its kept revision by number, waiting while that revision cannot be read. The test's starter gets a clock that moves. Builds: design 47 H H-A-286 (amending H-A-284; with H-A-283, H-A-285)
This commit is contained in:
@@ -177,7 +177,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now)
|
||||
starter.Asks, hearer.Asks = clock, clock
|
||||
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
|
||||
starter.Left = hearer.LeftInPlace // a same-source re-pin's superseded steps leave their runs (H-A-284)
|
||||
starter.Left = hearer.LeftInPlace // a same-source re-pin's superseded steps leave their runs (H-A-284)
|
||||
starter.Revision = widener.Revision // a kept mover is placed from its kept revision (H-A-286)
|
||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
|
||||
|
||||
@@ -324,7 +324,7 @@ func (h *Hearer) endUnanswered(key, ask, why string) error {
|
||||
var readers []int
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = sameValidate(&p, i, now)
|
||||
_, notAsked := releaseCompositions(&p, now)
|
||||
_, notAsked := releaseCompositions(&p, now, h.kept(key))
|
||||
readers = append(readers, notAsked...)
|
||||
}
|
||||
asked, err := askApproval(key, &p, now, h.Asker)
|
||||
|
||||
@@ -61,7 +61,18 @@ var moduleSteps = []string{seat.StepResultStepValidate, seat.StepResultStepCompo
|
||||
// placement makes the same id at every start and the bus dedupes on it, and each mover's step its own id (design 47 H
|
||||
// §746); started is the pipeline's start time, or a rerun's own (design 47 G A-297).
|
||||
func AskID(started time.Time, pipeline, module, artifact, step string) string {
|
||||
sum := sha256.Sum256([]byte(pipeline + "\n" + module + "\n" + artifact + "\n" + step))
|
||||
return keptAskID(started, pipeline, module, artifact, step, 0)
|
||||
}
|
||||
|
||||
// keptAskID is AskID of a step of a mover a same-source re-pin kept at catalog revision kept (H-A-286): the kept revision
|
||||
// is mixed into the id's sum, so no id equals one the pipeline asked for the mover's old fingerprint, and every placing,
|
||||
// pin-held ask, composition, rerun and restart of the kept mover finds the same ids again; kept 0 is AskID's own form.
|
||||
func keptAskID(started time.Time, pipeline, module, artifact, step string, kept int64) string {
|
||||
in := pipeline + "\n" + module + "\n" + artifact + "\n" + step
|
||||
if kept > 0 {
|
||||
in += "\nkept " + strconv.FormatInt(kept, 10)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(in))
|
||||
return strconv.FormatInt(started.UnixMilli(), 36) + hex.EncodeToString(sum[:3])
|
||||
}
|
||||
|
||||
@@ -80,14 +91,16 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
|
||||
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
|
||||
// work queues, one per running step.
|
||||
func Place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
|
||||
return place(p, m, reg, entries, now, asker, true)
|
||||
return place(p, m, reg, entries, now, asker, true, 0)
|
||||
}
|
||||
|
||||
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
|
||||
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
|
||||
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
|
||||
// naming the run's ask and the step that asked it (A-469m, A-469q).
|
||||
func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
|
||||
// kept is the catalog revision a same-source re-pin kept m at, 0 for a mover not kept: its every ask id mixes it in
|
||||
// (H-A-286).
|
||||
func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool, kept int64) ([]seat.StepResult, []Asked) {
|
||||
entry := entries[m.Module]
|
||||
candidate, named := CandidateOf(p)
|
||||
share = share && named
|
||||
@@ -99,7 +112,7 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
|
||||
// The build step is every mover's first, required whatever the policy names, asked of the seat mesh-builds for the
|
||||
// mover's own entry only (A-99, A-97): each mover is built by its own step (§746).
|
||||
build := seat.StepResult{Step: seat.StepResultStepBuild, Module: m.Module, Status: seat.StepResultStatusRequired, Fingerprint: m.Fingerprint,
|
||||
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(startOf(p, now), p.Pipeline, m.Module, "", seat.StepResultStepBuild)}
|
||||
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: keptAskID(startOf(p, now), p.Pipeline, m.Module, "", seat.StepResultStepBuild, kept)}
|
||||
steps = append(steps, build)
|
||||
asks = append(asks, buildAsk(p, m, build.Ask))
|
||||
for _, a := range arts {
|
||||
@@ -144,7 +157,7 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
|
||||
continue
|
||||
}
|
||||
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
|
||||
s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step)
|
||||
s.Ask = keptAskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step, kept)
|
||||
body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: body, Toolchain: a.Language, Commit: m.Commit}
|
||||
if share && BoundSeat(bs) {
|
||||
@@ -189,7 +202,7 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
|
||||
// validate is the assigner's, an accepted kind step of the seat mesh-wants, run once per take-in revision: every
|
||||
// mover's validate step reads the one ask, its id naming no mover (§745, H-A-51).
|
||||
s.Seat, s.Outcome, s.Started, s.Fingerprint = wants.Seat, seat.StepResultOutcomeRunning, &now, m.Fingerprint
|
||||
s.Ask = ValidateAskID(p, now)
|
||||
s.Ask = keptAskID(startOf(p, now), p.Pipeline, "", "", seat.StepResultStepValidate, kept)
|
||||
asks = append(asks, wantsAsk(p, m, s.Ask, wants.StepRunStepValidate))
|
||||
validating = true
|
||||
case seat.StepResultStepComposition:
|
||||
@@ -197,7 +210,7 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
|
||||
// runner's seat as its asker (H-A-88). It is asked once the mover's validate passed, at once when it has none
|
||||
// (§745).
|
||||
s.Seat, s.Fingerprint = wants.Seat, m.Fingerprint
|
||||
s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, "", step)
|
||||
s.Ask = keptAskID(startOf(p, now), p.Pipeline, m.Module, "", step, kept)
|
||||
if validating {
|
||||
s.Ask, s.Why = "", "asked once validate passed"
|
||||
break
|
||||
@@ -302,7 +315,7 @@ func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]
|
||||
return 0, nil
|
||||
}
|
||||
var entry catalog.CatalogEntry
|
||||
at, whose, unread := moverRevision(store, p, module)
|
||||
at, kept, whose, unread := moverRevision(store, p, module)
|
||||
switch half, err := readRevision(revision, p.Branch, at); {
|
||||
case unread != "":
|
||||
case err != nil:
|
||||
@@ -336,7 +349,7 @@ func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]
|
||||
}
|
||||
a := entry.Artifacts[j]
|
||||
st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now
|
||||
st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step)
|
||||
st.Ask = keptAskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step, kept)
|
||||
ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit}
|
||||
if a.Package != nil {
|
||||
ask.Package = a.Package.Name
|
||||
@@ -367,19 +380,45 @@ type keptRevisions interface {
|
||||
// moverRevision is the catalog revision mover module's entry of p is read at (H-A-284 (2)): the one its pipeline kept it at
|
||||
// when a same-source re-pin kept it, else the pipeline's; whose says which, and unread why it is not read when its record
|
||||
// cannot be.
|
||||
func moverRevision(store Updater, p seat.Pipeline, module string) (int64, string, string) {
|
||||
ks, ok := store.(keptRevisions)
|
||||
if !ok {
|
||||
return p.Revision, "the pipeline's", ""
|
||||
}
|
||||
kept, err := ks.KeptRevisions(p.Pipeline)
|
||||
// It answers that revision, the kept revision (0 for a mover not kept), whose it is, and why it is not read.
|
||||
func moverRevision(store Updater, p seat.Pipeline, module string) (int64, int64, string, string) {
|
||||
kept, err := keptOfStore(store, p.Pipeline)
|
||||
if err != nil {
|
||||
return p.Revision, "the pipeline's", fmt.Sprintf("the revision %s was kept at in %s cannot be read: %v", module, p.Pipeline, err)
|
||||
return p.Revision, 0, "the pipeline's", fmt.Sprintf("the revision %s was kept at in %s cannot be read: %v", module, p.Pipeline, err)
|
||||
}
|
||||
if r := kept[module]; r > 0 {
|
||||
return r, "the one the pipeline kept " + module + " at (H-A-284)", ""
|
||||
return r, r, "the one the pipeline kept " + module + " at (H-A-284)", ""
|
||||
}
|
||||
return p.Revision, "the pipeline's", ""
|
||||
return p.Revision, 0, "the pipeline's", ""
|
||||
}
|
||||
|
||||
// keptOfStore is pipeline key's kept-revisions in store (H-A-285); none for a store that keeps none.
|
||||
func keptOfStore(store any, key string) (map[string]int64, error) {
|
||||
ks, ok := store.(keptRevisions)
|
||||
if !ok || key == "" {
|
||||
return nil, nil
|
||||
}
|
||||
kept, err := ks.KeptRevisions(key)
|
||||
if errors.Is(err, ErrNoPipeline) {
|
||||
return nil, nil // a pipeline not written yet keeps none
|
||||
}
|
||||
return kept, err
|
||||
}
|
||||
|
||||
// keptEntries are the entries of branch's catalog revision rev, the one a same-source re-pin kept a mover at, read by its
|
||||
// number (H-A-286): a kept mover is placed and pin-checked from them, never from the branch's held view; false while they
|
||||
// cannot be read, and the mover waits.
|
||||
func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, bool) {
|
||||
if s.Revision == nil {
|
||||
s.logf("the runner reads no catalog revision, so a mover kept at revision %d of %s waits", rev, branch)
|
||||
return nil, false
|
||||
}
|
||||
half, err := s.Revision(branch, rev)
|
||||
if err != nil || half.Revision != rev {
|
||||
s.logf("revision %d of %s, which a mover is kept at, is not read (%v): the mover waits for it (H-A-286)", rev, branch, err)
|
||||
return nil, false
|
||||
}
|
||||
return half.Entries, true
|
||||
}
|
||||
|
||||
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
@@ -459,7 +498,8 @@ func wantsAsk(p seat.Pipeline, m seat.Mover, id, step string) Asked {
|
||||
// releaseCompositions asks the composition of each mover whose validate ended passed, and ends `not asked` the composition
|
||||
// of each whose validate did not pass, so a manifest the validator refuses is never composed (design 47 H §745). It answers
|
||||
// the asks to put and the indexes of the steps it ended.
|
||||
func releaseCompositions(p *seat.Pipeline, now time.Time) ([]Asked, []int) {
|
||||
// kept is p's kept-revisions: a kept mover's composition id mixes its kept revision in (H-A-286).
|
||||
func releaseCompositions(p *seat.Pipeline, now time.Time, kept map[string]int64) ([]Asked, []int) {
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
for _, m := range p.Movers {
|
||||
@@ -485,7 +525,7 @@ func releaseCompositions(p *seat.Pipeline, now time.Time) ([]Asked, []int) {
|
||||
ended = append(ended, c)
|
||||
continue
|
||||
}
|
||||
s.Ask = AskID(startOf(*p, now), p.Pipeline, m.Module, "", seat.StepResultStepComposition)
|
||||
s.Ask = keptAskID(startOf(*p, now), p.Pipeline, m.Module, "", seat.StepResultStepComposition, kept[m.Module])
|
||||
s.Outcome, s.Started, s.Why = seat.StepResultOutcomeRunning, &now, ""
|
||||
asks = append(asks, wantsAsk(*p, m, s.Ask, wants.StepRunStepComposition))
|
||||
}
|
||||
|
||||
@@ -45,8 +45,9 @@ func holdsOpen(p seat.Pipeline, half Half, module string) bool {
|
||||
// keepIn keeps in pipeline key each of movers it holds (H-A-283), answering the movers it does not hold. A kept mover whose
|
||||
// fingerprint the re-pin moved takes its new fingerprint and commit, and the pipeline keeps the revision it is from in its
|
||||
// kept-revisions (H-A-285); when its steps were placed, every one of them, open or ended, ends superseded and its steps are
|
||||
// placed again from the revision's entry, the build first (H-A-284 (1)), each under an ask of its own (placed at this
|
||||
// time), so no ask of the old fingerprint is kept for the new. One not placed yet is placed when it is due, as any mover.
|
||||
// placed again from the revision's entry, the build first (H-A-284 (1)), each under an ask id its kept revision is mixed
|
||||
// into (H-A-286), so no ask of the old fingerprint is kept for the new. One not placed yet is placed when it is due, from
|
||||
// the entries of the revision it is kept at.
|
||||
func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Mover {
|
||||
for {
|
||||
p, rev, err := s.Store.Get(key)
|
||||
@@ -164,7 +165,7 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
}
|
||||
|
||||
// placeAgain places the steps of kept mover m of p from the re-pin revision half's entry (H-A-284 (1)), as a mover is placed
|
||||
// (placeMovers), under asks of this placing's time: it answers the asks to put, the steps placed not asked because a
|
||||
// (placeMovers), under ask ids its kept revision is mixed into (H-A-286): it answers the asks to put, the steps placed not asked because a
|
||||
// dependency did not build, and whether m has no registration. A mover not due yet (it waits on a publish) is left
|
||||
// unplaced, placed when the publish is heard.
|
||||
func (s *Starter) placeAgain(p *seat.Pipeline, m seat.Mover, half Half, now time.Time) ([]Asked, []int, bool) {
|
||||
@@ -180,9 +181,7 @@ func (s *Starter) placeAgain(p *seat.Pipeline, m seat.Mover, half Half, now time
|
||||
s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", "))
|
||||
return nil, nil, false
|
||||
}
|
||||
at := *p
|
||||
at.Started = &now
|
||||
steps, asks := Place(at, m, r, half.Entries, now, s.Name)
|
||||
steps, asks := place(*p, m, r, half.Entries, now, s.Name, true, half.Revision)
|
||||
supersededWhy(*p, m.Module, half.Entries, steps)
|
||||
var notAsked []int
|
||||
if failed != "" {
|
||||
|
||||
@@ -3,7 +3,9 @@ package pipeline
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -12,12 +14,15 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// H-A-283, H-A-284, H-A-285: a re-pin revision of the same source moves no mover out of the running pipeline of that source:
|
||||
// the pipeline keeps it at its new fingerprint, every step of it ends superseded (said at the old fingerprint, its asks
|
||||
// dropped) and is placed again from the re-pin revision's entry under new asks, the build first, still naming the
|
||||
// pipeline's revision; the revision the mover is kept at rides in the record's kept-revisions, where the pin check reads
|
||||
// its entry; the re-pin starts no pipeline. A revision of a new source still starts its own and moves the mover out.
|
||||
func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
// ticking is a clock that moves a second at every read, so a start and a later re-pin never share a millisecond.
|
||||
func ticking() func() time.Time {
|
||||
at := time.Date(2026, 11, 21, 17, 0, 0, 0, time.UTC)
|
||||
return func() time.Time { at = at.Add(time.Second); return at }
|
||||
}
|
||||
|
||||
// repinFixture is a pipeline main.10 of source 41 moving app at src2:aa, started, with the re-pin revision 11 of the same
|
||||
// source moving app to src2:ab (an entry naming a package with no version yet), and a revision 12 of a new source 42.
|
||||
func repinFixture() (catalog.Registration, map[string]catalog.CatalogEntry, map[string]catalog.CatalogEntry, func(string, int64) (Half, error)) {
|
||||
reg, entries := placeFixture()
|
||||
repinned := map[string]catalog.CatalogEntry{}
|
||||
for k, e := range entries {
|
||||
@@ -29,11 +34,38 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
re := app
|
||||
re.Fp, re.Packages = "src2:ab", map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26", "git.novox.be/novox/x/go": ""}
|
||||
repinned["app"] = re
|
||||
read := func(branch string, revision int64) (Half, error) {
|
||||
switch revision {
|
||||
case 10:
|
||||
return Half{Revision: 10, Source: 41, Entries: entries}, nil
|
||||
case 11:
|
||||
return Half{Revision: 11, Source: 41, Entries: repinned, Call: "release of mesh-sdk 0.1.26",
|
||||
Movers: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}, nil
|
||||
case 12:
|
||||
return Half{Revision: 12, Source: 42, Entries: repinned, Call: "take-in of 42",
|
||||
Movers: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:ac", Repository: "o/app"}}}, nil
|
||||
}
|
||||
return Half{}, fmt.Errorf("no revision %d", revision)
|
||||
}
|
||||
return reg, entries, repinned, read
|
||||
}
|
||||
|
||||
func revisionEvent(n int64) catalog.CatalogRevisionEvent {
|
||||
return catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("main"), Revision: n, CallKind: "release"}
|
||||
}
|
||||
|
||||
// H-A-283, H-A-284, H-A-285, H-A-286: a re-pin revision of the same source moves no mover out of the running pipeline of that source:
|
||||
// the pipeline keeps it at its new fingerprint, every step of it ends superseded (said at the old fingerprint, its asks
|
||||
// dropped) and is placed again from the re-pin revision's entry under new asks, the build first, still naming the
|
||||
// pipeline's revision; the revision the mover is kept at rides in the record's kept-revisions, where the pin check reads
|
||||
// its entry; the re-pin starts no pipeline. A revision of a new source still starts its own and moves the mover out.
|
||||
func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
reg, entries, _, read := repinFixture()
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
var said []seat.StepEvent
|
||||
var left [][]seat.StepResult
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner",
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
Say: func(_ string, body any) error {
|
||||
if ev, ok := body.(seat.StepEvent); ok && ev.Result.Outcome == seat.StepResultOutcomeSuperseded {
|
||||
@@ -57,22 +89,7 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
asks.at, asks.ids, asks.asks = nil, nil, nil
|
||||
read := func(branch string, revision int64) (Half, error) {
|
||||
switch revision {
|
||||
case 10:
|
||||
return Half{Revision: 10, Source: 41, Entries: entries}, nil
|
||||
case 11:
|
||||
return Half{Revision: 11, Source: 41, Entries: repinned, Call: "release of mesh-sdk 0.1.26",
|
||||
Movers: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}, nil
|
||||
case 12:
|
||||
return Half{Revision: 12, Source: 42, Entries: repinned, Call: "take-in of 42",
|
||||
Movers: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:ac", Repository: "o/app"}}}, nil
|
||||
}
|
||||
return Half{}, fmt.Errorf("no revision %d", revision)
|
||||
}
|
||||
ev := func(n int64) catalog.CatalogRevisionEvent {
|
||||
return catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("main"), Revision: n, CallKind: "release"}
|
||||
}
|
||||
ev := revisionEvent
|
||||
if n, err := s.Revised(ev(11), read); n != 0 || err != nil {
|
||||
t.Fatalf("the same-source re-pin started %d pipelines (%v), want none", n, err)
|
||||
}
|
||||
@@ -123,3 +140,52 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
t.Errorf("the new source's revision left app in %s: %v", key, p.Superseded)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-286: a kept mover not placed at its keep (its registration arrived later) is placed from the revision it is kept at
|
||||
// while the branch's held view still holds the older revision, under ask ids its kept revision is mixed into; and a restart
|
||||
// (Resume) finds its running steps by those ids and asks them again.
|
||||
func TestAKeptMoverIsPlacedLateAndResumedAtItsKeptRevision(t *testing.T) {
|
||||
reg, entries, _, read := repinFixture()
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
registered := false
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Say: func(string, any) error { return nil },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
||||
key := Key("main", 10)
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
|
||||
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
|
||||
t.Fatalf("the re-pin started %d (%v)", n, err)
|
||||
}
|
||||
// The registration arrives while the runner's view still holds revision 10.
|
||||
registered = true
|
||||
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
|
||||
if n, err := s.Registered(store, reg, lagging); n != 1 || err != nil {
|
||||
t.Fatalf("the late registration placed in %d pipelines (%v)", n, err)
|
||||
}
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var build seat.StepResult
|
||||
held := 0
|
||||
for _, st := range p.Steps {
|
||||
switch {
|
||||
case st.Step == seat.StepResultStepBuild:
|
||||
build = st
|
||||
case pinHeld(st) && strings.HasSuffix(st.Why, "git.novox.be/novox/x/go"):
|
||||
held++
|
||||
}
|
||||
}
|
||||
if want := keptAskID(*p.Started, key, "app", "", seat.StepResultStepBuild, 11); build.Ask != want || build.Fingerprint != "src2:ab" || held == 0 {
|
||||
t.Fatalf("the late placing: build %+v (want ask %s), %d steps held on revision 11's unpinned package: %+v", build, want, held, p.Steps)
|
||||
}
|
||||
// A restart asks the kept mover's running steps again under the same ids.
|
||||
asks.at, asks.ids, asks.asks = nil, nil, nil
|
||||
if n, err := s.Resume(store, "main", Half{Revision: 10, Source: 41, Entries: entries}); n == 0 || err != nil || !slices.Contains(asks.ids, build.Ask) {
|
||||
t.Fatalf("Resume asked %d (%v), ids %v: want the kept build %s again", n, err, asks.ids, build.Ask)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +168,10 @@ type Starter struct {
|
||||
// Left takes the steps a same-source re-pin superseded in a pipeline that kept their mover out of the test runs they
|
||||
// read and drops their own asks (H-A-283, H-A-284 (1); Hearer.LeftInPlace); nil leaves them.
|
||||
Left func(key string, leave, drop []seat.StepResult, why string)
|
||||
// Revision reads a catalog revision of a branch by its number through the catalog seat's revision verb: a mover a
|
||||
// same-source re-pin kept is placed from the entries of the revision it is kept at (H-A-286); nil leaves such a mover
|
||||
// waiting.
|
||||
Revision func(branch string, revision int64) (Half, error)
|
||||
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
||||
Fills *Fills
|
||||
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
||||
@@ -337,6 +341,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
var unregistered []string
|
||||
var notAsked []int
|
||||
var waits []seat.Mover
|
||||
kept, err := keptOfStore(s.Store, p.Pipeline)
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
|
||||
}
|
||||
all := entries
|
||||
for _, m := range p.Movers {
|
||||
if placed(*p, m.Module) {
|
||||
continue
|
||||
@@ -346,6 +355,13 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
unregistered = append(unregistered, m.Module)
|
||||
continue
|
||||
}
|
||||
// A mover a same-source re-pin kept is placed from the revision it is kept at, its ids its own (H-A-286).
|
||||
entries := all
|
||||
if rev := kept[m.Module]; rev > 0 {
|
||||
if entries, ok = s.keptEntries(p.Branch, rev); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard,
|
||||
// ended not asked when one's build ended without passing.
|
||||
due, waiting, failed := ready(*p, m.Module, entries)
|
||||
@@ -354,7 +370,7 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
waits = append(waits, m)
|
||||
continue
|
||||
}
|
||||
steps, a := Place(*p, m, r, entries, now, s.Name)
|
||||
steps, a := place(*p, m, r, entries, now, s.Name, true, kept[m.Module])
|
||||
supersededWhy(*p, m.Module, entries, steps)
|
||||
for _, st := range steps {
|
||||
if pinHeld(st) {
|
||||
@@ -393,13 +409,13 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
}
|
||||
}
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, entries, now, s.Name); ended {
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, all, now, s.Name); ended {
|
||||
return nil, unregistered, steps, notAsked
|
||||
} else {
|
||||
notAsked = append(notAsked, steps...)
|
||||
}
|
||||
// A mover placed after the validate ended has its composition asked, or not asked, at once (§745).
|
||||
more, _ := releaseCompositions(p, now)
|
||||
more, _ := releaseCompositions(p, now, kept)
|
||||
asks = append(asks, more...)
|
||||
asks, ended := s.holderless(p, asks, now)
|
||||
return asks, unregistered, ended, notAsked
|
||||
@@ -614,13 +630,24 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
|
||||
}
|
||||
}
|
||||
var due []Asked
|
||||
kept, err := keptOfStore(store, key)
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its kept movers' steps are not asked again: %v", key, err)
|
||||
}
|
||||
for _, m := range p.Movers {
|
||||
reg, ok := s.Registration(m.Module)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
_, shared := Place(p, m, reg, half.Entries, now, s.Name)
|
||||
_, unshared := place(p, m, reg, half.Entries, now, s.Name, false)
|
||||
// A kept mover's steps are found by the ids its kept revision gives, from that revision's entries (H-A-286).
|
||||
entries, rev := half.Entries, kept[m.Module]
|
||||
if rev > 0 {
|
||||
if entries, ok = s.keptEntries(p.Branch, rev); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
_, shared := place(p, m, reg, entries, now, s.Name, true, rev)
|
||||
_, unshared := place(p, m, reg, entries, now, s.Name, false, rev)
|
||||
for _, a := range shared {
|
||||
if running[a.ID] && !own[a.ID] {
|
||||
due = append(due, a)
|
||||
|
||||
@@ -199,6 +199,15 @@ func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.Tes
|
||||
return first
|
||||
}
|
||||
|
||||
// kept is pipeline key's kept-revisions (H-A-285, H-A-286); none when they cannot be read, said.
|
||||
func (h *Hearer) kept(key string) map[string]int64 {
|
||||
kept, err := keptOfStore(h.Store, key)
|
||||
if err != nil {
|
||||
h.logf("the kept-revisions of %s are not read, so its kept movers' asks take the ids of movers not kept: %v", key, err)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
func (h *Hearer) now() time.Time {
|
||||
if h.Now != nil {
|
||||
return h.Now().UTC()
|
||||
@@ -314,7 +323,7 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = append(readers, sameValidate(&p, i, now)...)
|
||||
var notAsked []int
|
||||
due, notAsked = releaseCompositions(&p, now)
|
||||
due, notAsked = releaseCompositions(&p, now, h.kept(ev.Pipeline))
|
||||
readers = append(readers, notAsked...)
|
||||
}
|
||||
from := now
|
||||
|
||||
@@ -530,9 +530,11 @@ func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
// under one id is kept once), saying it is a rerun and by whom. The validate ask is the pipeline's one, naming no mover
|
||||
// (H-A-51): its rerun asks that one ask again; a composition's rerun builds the mover's composition ask (H-A-236).
|
||||
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
|
||||
id := AskID(now, p.Pipeline, s.Module, s.Artifact, s.Step)
|
||||
// A kept mover's rerun mixes its kept revision into the id (H-A-286).
|
||||
rev := v.keptOf(p.Pipeline)[s.Module]
|
||||
id := keptAskID(now, p.Pipeline, s.Module, s.Artifact, s.Step, rev)
|
||||
if s.Seat == wants.Seat && s.Step == seat.StepResultStepValidate {
|
||||
id = AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate)
|
||||
id = keptAskID(now, p.Pipeline, "", "", seat.StepResultStepValidate, rev)
|
||||
}
|
||||
a, err := v.placedAsk(p, s, id, now)
|
||||
if err != nil {
|
||||
@@ -580,8 +582,15 @@ func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time
|
||||
} else {
|
||||
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
|
||||
}
|
||||
_, placed := place(p, m, reg, half.Entries, now, v.Starter.Name, false)
|
||||
base := AskID(startOf(p, now), p.Pipeline, m.Module, s.Artifact, s.Step)
|
||||
// A kept mover's step is the one placed from the revision it is kept at, under its kept ids (H-A-286).
|
||||
entries, rev := half.Entries, v.keptOf(p.Pipeline)[m.Module]
|
||||
if rev > 0 {
|
||||
if entries, ok = v.Starter.keptEntries(p.Branch, rev); !ok {
|
||||
return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read: the step is asked when it is", rev, p.Branch, m.Module)
|
||||
}
|
||||
}
|
||||
_, placed := place(p, m, reg, entries, now, v.Starter.Name, false, rev)
|
||||
base := keptAskID(startOf(p, now), p.Pipeline, m.Module, s.Artifact, s.Step, rev)
|
||||
for _, a := range placed {
|
||||
if a.ID != base {
|
||||
continue
|
||||
@@ -596,6 +605,14 @@ func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time
|
||||
return nil, fmt.Errorf("the %s step of %s is placed no ask now", s.Step, p.Pipeline)
|
||||
}
|
||||
|
||||
// keptOf is pipeline key's kept-revisions (H-A-285); none when they cannot be read, said.
|
||||
func (v Verbs) keptOf(key string) map[string]int64 {
|
||||
if v.Hearer == nil {
|
||||
return nil
|
||||
}
|
||||
return v.Hearer.kept(key)
|
||||
}
|
||||
|
||||
// Run starts a run on request for the caller the bus proved (design 47 G §64; Starter.Run): its pipelines written, its
|
||||
// run-started said, its steps placed when its candidate's revision is heard.
|
||||
func (v Verbs) Run(caller string, in seat.RunInput) (seat.RunAnswer, error) {
|
||||
|
||||
Reference in New Issue
Block a user