H-A-479 second addendum: own movers withdrawn, no side-by-side start, unbounded n
A pipeline ending not passed withdrew every reservation of its revision, so
an again pipeline's failure could withdraw the head's take-in pipeline's
reservations and the other way round. The indexer now withdraws only the
reservations of the ended pipeline's own movers, read from the movers its
pipeline-ended carries, and withdraws the revision's gathering only when
none is left. A start with modules is refused for a module whose head entry
a pipeline still runs ("its pipeline <key> is running"), and takes the next
n without a bound. A rerun reopening an ended pipeline says its
pipeline-started before the step is asked again, so the revision is pinned
before anything reads it. Tests now cover the again form through the
branch's key forms and a release written into a passed again pipeline.
Builds: design 47 H2 H-A-479 second addendum, H-A-479 addendum (2); review round 1
This commit is contained in:
@@ -105,9 +105,14 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
}
|
||||
}
|
||||
// A pipeline that did not pass withdraws the reservations its revision holds (design 47 H2 §805, H-A-418 (1)).
|
||||
// A pipeline that did not pass withdraws the reservations its revision holds of its own movers, never another open
|
||||
// pipeline's at that revision (design 47 H2 §805, H-A-418 (1), H-A-479 second addendum).
|
||||
if ended.Outcome != runner.PipelineEndedOutcomePassed {
|
||||
if err := ix.Withdraw(ctx, ended.Branch, ended.Revision); err != nil {
|
||||
movers := make([]string, 0, len(ended.Movers))
|
||||
for _, m := range ended.Movers {
|
||||
movers = append(movers, m.Module)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, ended.Branch, ended.Revision, movers); err != nil {
|
||||
return fmt.Errorf("the reservations of %s are not withdrawn: %w", BranchObject(ended.Branch, ended.Revision), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,7 +151,14 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 4 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want 4", err, newest())
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3); err != nil || newest() != 5 {
|
||||
// H-A-479 second addendum: a pipeline at the same revision whose movers are others withdraws none of mesh-sdk's.
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 4 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 4: nothing of mesh-sdk withdrawn", err, newest())
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); !kept {
|
||||
t.Fatalf("another pipeline's withdrawal took mesh-sdk's reservation")
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 5 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 5", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
|
||||
@@ -350,20 +350,35 @@ func (ix *Indexer) settleNow(ctx context.Context, revision string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds is withdrawn, its
|
||||
// series' numbered set back to the newest released version, its importers re-pinned to that, and its key deleted.
|
||||
func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64) error {
|
||||
// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds of one of the
|
||||
// ended pipeline's own movers is withdrawn, its series' numbered set back to the newest released version, its importers
|
||||
// re-pinned to that, and its key deleted. Another open pipeline's reservations at the same revision are kept, and the
|
||||
// revision's gathering is withdrawn only when none of them is left (design 47 H2 H-A-479 second addendum).
|
||||
func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64, movers []string) error {
|
||||
if ix.Heard == nil {
|
||||
return nil
|
||||
}
|
||||
name := BranchObject(branch, revision)
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found {
|
||||
_ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it
|
||||
}
|
||||
held, revs, err := ix.reservedOf(ctx, name)
|
||||
if err != nil || len(held) == 0 {
|
||||
all, revs, err := ix.reservedOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held := map[string]Reserved{}
|
||||
for key, r := range all {
|
||||
if slices.Contains(movers, r.Module) {
|
||||
held[key] = r
|
||||
} else {
|
||||
delete(revs, key)
|
||||
}
|
||||
}
|
||||
if len(held) == len(all) {
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found {
|
||||
_ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it
|
||||
}
|
||||
}
|
||||
if len(held) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, r := range held {
|
||||
for _, one := range r.Reservations {
|
||||
if err := ix.withdrawNumbered(ctx, one); err != nil {
|
||||
|
||||
@@ -295,14 +295,17 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
a, err := v.rerun(caller, in)
|
||||
if err == nil && reopens && v.Starter != nil {
|
||||
if p, _, gerr := v.Hearer.Store.Get(in.Pipeline); gerr == nil && p.Ended == nil {
|
||||
v.Starter.sayStartedPipeline(p, v.now())
|
||||
v.Starter.logf("%s reopened by a rerun: its pipeline-started is said again, so its revision %d is pinned again (H-A-479)", in.Pipeline, p.Revision)
|
||||
}
|
||||
return v.rerun(caller, in, reopens)
|
||||
}
|
||||
|
||||
// sayReopened says pipeline-started again for p, written reopened by a rerun, before its step is asked again, so the
|
||||
// indexer pins its revision again before anything reads it (H-A-479 addendum (2)).
|
||||
func (v Verbs) sayReopened(p seat.Pipeline, reopens bool) {
|
||||
if !reopens || v.Starter == nil || p.Ended != nil {
|
||||
return
|
||||
}
|
||||
return a, err
|
||||
v.Starter.sayStartedPipeline(p, v.now())
|
||||
v.Starter.logf("%s reopened by a rerun: its pipeline-started is said again, so its revision %d is pinned again (H-A-479)", p.Pipeline, p.Revision)
|
||||
}
|
||||
|
||||
// reopening says whether a rerun of pipeline key reopens an ended delivering pipeline, refusing it when its revision is
|
||||
@@ -324,10 +327,10 @@ func (v Verbs) reopening(key string) (bool, error) {
|
||||
}
|
||||
|
||||
// rerun asks one step again (Rerun).
|
||||
func (v Verbs) rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error) {
|
||||
func (v Verbs) rerun(caller string, in seat.RerunInput, reopens bool) (seat.RerunAnswer, error) {
|
||||
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
|
||||
if i, err := stepOf(p, in.Module, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
|
||||
return v.rerunTest(caller, in, p.Steps[i])
|
||||
return v.rerunTest(caller, in, p.Steps[i], reopens)
|
||||
}
|
||||
}
|
||||
var ask *Asked
|
||||
@@ -378,6 +381,7 @@ func (v Verbs) rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
v.sayReopened(p, reopens)
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
|
||||
@@ -395,7 +399,7 @@ func (v Verbs) rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
|
||||
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
|
||||
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult, reopens bool) (seat.RerunAnswer, error) {
|
||||
h := v.Hearer
|
||||
run := sharedRun(st)
|
||||
if open(st) {
|
||||
@@ -476,6 +480,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
v.sayReopened(p, reopens)
|
||||
switch {
|
||||
case join || read:
|
||||
v.Starter.joinRun(run, reader)
|
||||
@@ -734,9 +739,6 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
|
||||
return seat.StartAnswer{Pipeline: key}, nil
|
||||
}
|
||||
|
||||
// againTries bounds how many next numbers a start again tries when another start took the one it read (H-A-479).
|
||||
const againTries = 10
|
||||
|
||||
// startAgain starts by hand, recorded in the hand-act log first, a pipeline at the head of a watched branch carrying the
|
||||
// named modules whose newest entry, the head's, has no passed pipeline (a failed, cut or superseded one), with the head's
|
||||
// revision as its candidate and the take-in's tiers (design 47 H2 H-A-479): keyed `<branch>.<revision>.again.<n>`, n from 1,
|
||||
@@ -767,13 +769,16 @@ func (v Verbs) startAgain(in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if len(absent) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the head of %s (revision %d) holds no entry of %s: nothing was started", in.Branch, half.Revision, strings.Join(absent, ", "))
|
||||
}
|
||||
passed, err := v.passedEntries(in.Branch, movers)
|
||||
passed, running, err := v.heldEntries(in.Branch, movers)
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
if len(passed) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the newest entry of %s passed already: nothing was started", strings.Join(passed, ", "))
|
||||
}
|
||||
if len(running) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the newest entry of %s: nothing was started", strings.Join(running, ", "))
|
||||
}
|
||||
movers = Tiers(movers, half.Entries, v.Starter.logf)
|
||||
names := make([]string, 0, len(movers))
|
||||
for _, m := range movers {
|
||||
@@ -794,7 +799,8 @@ func (v Verbs) startAgain(in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if err := v.record(seat.VerbStart, fmt.Sprintf("started %s again at the head of %s by hand", strings.Join(names, ", "), in.Branch), in.Why); err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
for range againTries {
|
||||
// The next n is taken without a bound: each refusal means the key exists (H-A-479 second addendum).
|
||||
for ; ; n++ {
|
||||
key := AgainPipelineKey(in.Branch, half.Revision, n)
|
||||
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries,
|
||||
Key: key, Movers: movers})
|
||||
@@ -805,26 +811,26 @@ func (v Verbs) startAgain(in seat.StartInput) (seat.StartAnswer, error) {
|
||||
v.Starter.logf("%s starts %s again at the head by hand (H-A-479)", key, strings.Join(names, ", "))
|
||||
return seat.StartAnswer{Pipeline: key}, nil
|
||||
}
|
||||
n++
|
||||
}
|
||||
return seat.StartAnswer{}, fmt.Errorf("no key %s<n> was free in %d tries: nothing was started", prefix, againTries)
|
||||
}
|
||||
|
||||
// passedEntries are the movers whose module, at the mover's fingerprint, a passed delivering pipeline of branch carries and
|
||||
// did not leave (H-A-479: a newest entry that passed is not started again).
|
||||
func (v Verbs) passedEntries(branch string, movers []seat.Mover) ([]string, error) {
|
||||
// heldEntries are the movers whose module, at the mover's fingerprint, a delivering pipeline of branch carries and did not
|
||||
// leave: passed, those a passed pipeline holds (a newest entry that passed is not started again, H-A-479); running, those a
|
||||
// pipeline not ended holds, each said "<module>: its pipeline <key> is running", so two pipelines never build one entry side
|
||||
// by side (H-A-479 second addendum).
|
||||
func (v Verbs) heldEntries(branch string, movers []seat.Mover) (passed, running []string, err error) {
|
||||
pkeys, err := branchKeys(v.Hearer.Store, branch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pipelines of %s are not listed, so nothing was started: %w", branch, err)
|
||||
return nil, nil, fmt.Errorf("the pipelines of %s are not listed, so nothing was started: %w", branch, err)
|
||||
}
|
||||
var passed []string
|
||||
seen := map[string]bool{}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pipeline %s is not read, so nothing was started: %w", key, err)
|
||||
return nil, nil, fmt.Errorf("the pipeline %s is not read, so nothing was started: %w", key, err)
|
||||
}
|
||||
if p.Run != "" || p.Outcome != seat.PipelineOutcomePassed {
|
||||
open := p.Ended == nil && (p.Outcome == seat.PipelineOutcomeRunning || p.Outcome == seat.PipelineOutcomeWaiting)
|
||||
if p.Run != "" || (p.Outcome != seat.PipelineOutcomePassed && !open) {
|
||||
continue
|
||||
}
|
||||
for _, m := range movers {
|
||||
@@ -833,11 +839,15 @@ func (v Verbs) passedEntries(branch string, movers []seat.Mover) ([]string, erro
|
||||
}
|
||||
if pm, ok := MoverOf(p, m.Module); ok && pm.Fingerprint == m.Fingerprint {
|
||||
seen[m.Module] = true
|
||||
passed = append(passed, m.Module+" (in "+key+")")
|
||||
if open {
|
||||
running = append(running, m.Module+": its pipeline "+key+" is running")
|
||||
} else {
|
||||
passed = append(passed, m.Module+" (in "+key+")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return passed, nil
|
||||
return passed, running, nil
|
||||
}
|
||||
|
||||
// AskAgain asks a declined approval step once more (§306, §36), recorded in the hand-act log first: the step waits again,
|
||||
|
||||
@@ -459,6 +459,16 @@ func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) {
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Modules: []string{"app"}, Why: "x"}); err == nil {
|
||||
t.Error("a start naming both module and modules was taken")
|
||||
}
|
||||
// A pipeline still running at app's head entry refuses the start: two pipelines never build one entry side by side.
|
||||
store["main.7"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.7", Branch: "main", Source: 6, Revision: 7, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "app", Fingerprint: "src2:aa"}}}, 1}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "its pipeline main.7 is running") {
|
||||
t.Errorf("a start of app while main.7 runs it = %v; want refused naming main.7", err)
|
||||
}
|
||||
delete(store, "main.7")
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app", "lib"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "lib") {
|
||||
t.Errorf("a start naming lib, whose newest entry passed, = %v; want refused naming lib", err)
|
||||
}
|
||||
@@ -479,7 +489,9 @@ func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) {
|
||||
if len(started) != 1 || started[0].Pipeline != "main.9.again.1" || started[0].Revision != 9 {
|
||||
t.Errorf("pipeline-started = %+v, want main.9.again.1 at revision 9 (its revision pinned)", started)
|
||||
}
|
||||
// The next start at the same head takes the next n.
|
||||
// The next start at the same head, once main.9.again.1 failed, takes the next n.
|
||||
at := time.Unix(3, 0)
|
||||
p.Outcome, p.Ended = seat.PipelineOutcomeFailed, &at
|
||||
store["main.9.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
@@ -490,6 +502,20 @@ func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) {
|
||||
if len(*acts) != 2 {
|
||||
t.Errorf("hand acts %v, want the two starts recorded", *acts)
|
||||
}
|
||||
// main.9.again.1 passed: its version is written into it as into any pipeline of its source, and a further start of app is
|
||||
// refused naming it, read through the branch's key forms.
|
||||
p.Outcome = seat.PipelineOutcomePassed
|
||||
store["main.9.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
if wrote, err := v.Hearer.Released(catalog.VersionReleasedEvent{Module: "app", Version: "0.1.2", Source: 7}, "main"); err != nil || wrote != 1 ||
|
||||
store["main.9.again.1"].p.Movers[0].Version != "0.1.2" {
|
||||
t.Errorf("the release of app 0.1.2 wrote %d (%v), the again pipeline's mover %+v; want its version written", wrote, err, store["main.9.again.1"].p.Movers)
|
||||
}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "main.9.again.1") {
|
||||
t.Errorf("a start of app after main.9.again.1 passed = %v; want refused naming it", err)
|
||||
}
|
||||
if !IsStepsKey(StepsKey("main.9.again.1", "app")) {
|
||||
t.Error("a mover key of an again pipeline is not read as a mover's steps")
|
||||
}
|
||||
@@ -502,6 +528,9 @@ func TestARerunOfAnEndedPipelinePinsItsRevisionAgainOrIsRefusedWhenItIsGone(t *t
|
||||
var started []seat.PipelineStarted
|
||||
v.Starter.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
if len(asks.asks) != 0 {
|
||||
t.Errorf("pipeline-started said after the step was asked again (%d asks): its revision is pinned too late", len(asks.asks))
|
||||
}
|
||||
started = append(started, ps)
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user