Design 47 H1: a moved mover leaves a running pipeline too (H-A-223)
A release re-pinned node-tools while main.54 still built it at its old fingerprint; the build failed and failed the pipeline for its other 119 movers, because supersession took movers out of waiting pipelines only. Every start of a newer revision's pipeline now also walks the older running pipelines of its branch: each mover whose fingerprint it moved leaves, its open steps ending superseded with a why naming the newer pipeline, a step reading a test run leaving it as a cancel's does, and the mover kept in the pipeline's superseded. The end judgement counts a left mover's steps as neither holding nor failed, so the pipeline ends by its other movers alone; one every mover left ends superseded. A run on request's pipeline is never touched. Builds: design 47 H H-A-223, H-A-163, H-A-213
This commit is contained in:
@@ -160,6 +160,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||||
hearer.AskSteps = starter.AskSteps
|
hearer.AskSteps = starter.AskSteps
|
||||||
|
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
|
||||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||||
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
|
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
|
||||||
|
|||||||
@@ -162,6 +162,9 @@ type Starter struct {
|
|||||||
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
|
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
|
||||||
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
|
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
|
||||||
Ended func(p seat.Pipeline, steps []int) error
|
Ended func(p seat.Pipeline, steps []int) error
|
||||||
|
// Supersede takes out of every older running pipeline of newer's branch each mover whose fingerprint newer moved
|
||||||
|
// (H-A-223; Hearer.SupersedeRunning); nil leaves running pipelines as they are.
|
||||||
|
Supersede func(newer seat.Pipeline)
|
||||||
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
||||||
Fills *Fills
|
Fills *Fills
|
||||||
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
||||||
@@ -538,6 +541,10 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
|||||||
// A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its
|
// A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its
|
||||||
// wait on its turn.
|
// wait on its turn.
|
||||||
s.sayStartedPipeline(p, now)
|
s.sayStartedPipeline(p, now)
|
||||||
|
// A mover whose fingerprint this revision moved leaves every older running pipeline of the branch (H-A-223).
|
||||||
|
if s.Supersede != nil {
|
||||||
|
s.Supersede(p)
|
||||||
|
}
|
||||||
if !place {
|
if !place {
|
||||||
s.sayQueueWait(seat.EventWaitHeld, p, now, "")
|
s.sayQueueWait(seat.EventWaitHeld, p, now, "")
|
||||||
if _, err := s.supersedeWaiting(p); err != nil {
|
if _, err := s.supersedeWaiting(p); err != nil {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -611,9 +612,16 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
|
|||||||
}
|
}
|
||||||
// A mover with no step placed waits on its registration or a publish: it holds the end only while every placed mover's
|
// A mover with no step placed waits on its registration or a publish: it holds the end only while every placed mover's
|
||||||
// required steps passed, since once one failed or could not run the pipeline can no longer pass (H-A-163, H-A-171).
|
// required steps passed, since once one failed or could not run the pipeline can no longer pass (H-A-163, H-A-171).
|
||||||
|
// A mover that left for a newer pipeline (H-A-223) holds nothing and fails nothing: its steps are not judged, and a
|
||||||
|
// pipeline every mover left ends superseded.
|
||||||
|
if allLeft(*p) {
|
||||||
|
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeSuperseded, "every mover left it for a newer pipeline (H-A-223)", &now
|
||||||
|
p.EndCount++
|
||||||
|
return true
|
||||||
|
}
|
||||||
var waiting []string
|
var waiting []string
|
||||||
for _, m := range p.Movers {
|
for _, m := range p.Movers {
|
||||||
if !placed(*p, m.Module) {
|
if !placed(*p, m.Module) && !slices.Contains(p.Superseded, m.Module) {
|
||||||
waiting = append(waiting, m.Module)
|
waiting = append(waiting, m.Module)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -621,6 +629,8 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
|
|||||||
var notAsked []int
|
var notAsked []int
|
||||||
for i, s := range p.Steps {
|
for i, s := range p.Steps {
|
||||||
switch {
|
switch {
|
||||||
|
case slices.Contains(p.Superseded, s.Module):
|
||||||
|
continue
|
||||||
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
|
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
|
||||||
notAsked = append(notAsked, i)
|
notAsked = append(notAsked, i)
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -377,3 +377,59 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
|||||||
t.Errorf("recovered: said %v", said)
|
t.Errorf("recovered: said %v", said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A mover whose fingerprint a newer revision moved leaves a running pipeline (H-A-223, H-A-163): its open steps end
|
||||||
|
// superseded, never failed, the other mover's go on and the pipeline ends by them alone; one every mover left ends
|
||||||
|
// superseded.
|
||||||
|
func TestAMovedMoverLeavesARunningPipeline(t *testing.T) {
|
||||||
|
t0 := time.Date(2026, 11, 21, 1, 0, 0, 0, time.UTC)
|
||||||
|
running := func() seat.Pipeline {
|
||||||
|
return seat.Pipeline{V: 1, Pipeline: "main.54", Branch: "main", Source: 54, Revision: 54, Outcome: seat.PipelineOutcomeRunning, Started: &t0,
|
||||||
|
Movers: []seat.Mover{{Module: "a", Commit: "c1", Fingerprint: "src2:a1"}, {Module: "b", Commit: "c1", Fingerprint: "src2:b1"}},
|
||||||
|
Steps: []seat.StepResult{
|
||||||
|
{Module: "a", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x1", Started: &t0},
|
||||||
|
{Module: "b", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x2", Started: &t0},
|
||||||
|
{Module: "b", Step: seat.StepResultStepLint, Artifact: "b:program", Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x3", Started: &t0}}}
|
||||||
|
}
|
||||||
|
rows := []struct {
|
||||||
|
name string
|
||||||
|
newer []seat.Mover
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"a newer revision moving b: b's steps end superseded, a's go on and end the pipeline passed alone",
|
||||||
|
[]seat.Mover{{Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}, "running; a build running; b build superseded; b lint superseded | passed"},
|
||||||
|
{"a newer revision moving every mover ends the pipeline superseded",
|
||||||
|
[]seat.Mover{{Module: "a", Commit: "c2", Fingerprint: "src2:a2"}, {Module: "b", Commit: "c2", Fingerprint: "src2:b2"}},
|
||||||
|
"superseded; a build superseded; b build superseded; b lint superseded | superseded"},
|
||||||
|
{"a newer revision at the same fingerprint takes nothing", []seat.Mover{{Module: "b", Commit: "c1", Fingerprint: "src2:b1"}},
|
||||||
|
"running; a build running; b build running; b lint running | running"},
|
||||||
|
}
|
||||||
|
for _, r := range rows {
|
||||||
|
t.Run(r.name, func(t *testing.T) {
|
||||||
|
store := memStore{}
|
||||||
|
store["main.54"] = struct {
|
||||||
|
p seat.Pipeline
|
||||||
|
rev uint64
|
||||||
|
}{running(), 1}
|
||||||
|
var said []string
|
||||||
|
h := &Hearer{Store: store, Say: func(subject string, _ any) error { said = append(said, subject); return nil }, Now: func() time.Time { return t0 }}
|
||||||
|
h.SupersedeRunning(seat.Pipeline{Pipeline: "main.57", Branch: "main", Revision: 57, Movers: r.newer})
|
||||||
|
p, _, _ := store.Get("main.54")
|
||||||
|
got := []string{p.Outcome}
|
||||||
|
for _, s := range p.Steps {
|
||||||
|
got = append(got, s.Module+" "+s.Step+" "+s.Outcome)
|
||||||
|
if s.Outcome == seat.StepResultOutcomeSuperseded && !strings.Contains(s.Why, "main.57") {
|
||||||
|
t.Errorf("the superseded step's why %q names no newer pipeline", s.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The mover left holds nothing: a's build passing ends the pipeline by a alone.
|
||||||
|
if p.Outcome == seat.PipelineOutcomeRunning {
|
||||||
|
p.Steps[0].Outcome, p.Steps[0].Ended = seat.StepResultOutcomePassed, &t0
|
||||||
|
endPipeline(&p, t0, nil)
|
||||||
|
}
|
||||||
|
if s := strings.Join(got, "; ") + " | " + p.Outcome; s != r.want {
|
||||||
|
t.Errorf("got %s\nwant %s", s, r.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||||
|
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder
|
// Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder
|
||||||
@@ -50,10 +51,106 @@ func (h *Hearer) Taken(ask, node string) error {
|
|||||||
|
|
||||||
func ptrTime(t time.Time) *time.Time { return &t }
|
func ptrTime(t time.Time) *time.Time { return &t }
|
||||||
|
|
||||||
|
// allLeft says whether every mover of p left it for a newer pipeline (H-A-223).
|
||||||
|
func allLeft(p seat.Pipeline) bool {
|
||||||
|
if len(p.Movers) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, m := range p.Movers {
|
||||||
|
if !slices.Contains(p.Superseded, m.Module) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupersedeRunning takes out of every older running pipeline of newer's branch each mover whose fingerprint newer moved
|
||||||
|
// (H-A-223): its open steps end superseded, naming newer, never failed; a step reading a test run leaves it, as a cancel's
|
||||||
|
// does; the mover is kept in the pipeline's superseded, so the end judgement (H-A-163) counts it as left and newer carries
|
||||||
|
// it. A pipeline then judged ends as at a step's end, by its other movers alone; one every mover left ends superseded.
|
||||||
|
// A run on request's pipeline is never touched. It answers the pipelines it touched.
|
||||||
|
func (h *Hearer) SupersedeRunning(newer seat.Pipeline) []string {
|
||||||
|
pkeys, err := allKeys(h.Store)
|
||||||
|
if err != nil {
|
||||||
|
h.logf("the running pipelines are not listed, so none is superseded by %s: %v", newer.Pipeline, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var touched []string
|
||||||
|
for _, key := range pkeys {
|
||||||
|
if key == newer.Pipeline {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.supersedeRunningOne(key, newer) {
|
||||||
|
touched = append(touched, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return touched
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool {
|
||||||
|
for {
|
||||||
|
p, rev, err := h.Store.Get(key)
|
||||||
|
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || p.Branch != newer.Branch || p.Run != "" || newer.Run != "" ||
|
||||||
|
p.Revision >= newer.Revision {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var moved []string
|
||||||
|
for _, m := range p.Movers {
|
||||||
|
if slices.Contains(p.Superseded, m.Module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if nm, ok := MoverOf(newer, m.Module); ok && nm.Fingerprint != m.Fingerprint {
|
||||||
|
moved = append(moved, m.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(moved) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
now := h.now()
|
||||||
|
why := "superseded: the newer " + newer.Pipeline + " moved its fingerprint, and carries it (H-A-223)"
|
||||||
|
var ended []int
|
||||||
|
var leave []seat.StepResult
|
||||||
|
for i := range p.Steps {
|
||||||
|
st := &p.Steps[i]
|
||||||
|
if !slices.Contains(moved, st.Module) || (!open(*st) && st.Outcome != seat.StepResultOutcomeWaiting) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if sharedRun(*st) != "" && st.Seat != wants.Seat {
|
||||||
|
leave = append(leave, *st)
|
||||||
|
}
|
||||||
|
st.Outcome, st.Why, st.Ended = seat.StepResultOutcomeSuperseded, why, &now
|
||||||
|
p.Unlogged = append(p.Unlogged, endEntry(key, *st, now))
|
||||||
|
ended = append(ended, i)
|
||||||
|
}
|
||||||
|
p.Superseded = append(p.Superseded, moved...)
|
||||||
|
done := h.endPipeline(&p, now)
|
||||||
|
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
h.logf("the running pipeline %s is not written, so its movers moved by %s stay: %v", key, newer.Pipeline, err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
h.logf("%s: %s moved by the newer %s leave it, %d of its open steps ended superseded (H-A-223)", key, strings.Join(moved, ", "), newer.Pipeline, len(ended))
|
||||||
|
for _, i := range ended {
|
||||||
|
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil {
|
||||||
|
h.logf("step-ended of %s's %s %s, superseded, is not said: %v", key, p.Steps[i].Module, p.Steps[i].Step, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, st := range leave {
|
||||||
|
h.leaveRun(key, st, "the mover "+st.Module+" left "+key+" for the newer "+newer.Pipeline)
|
||||||
|
}
|
||||||
|
if err := h.said(p, nil, done); err != nil {
|
||||||
|
h.logf("%v", err)
|
||||||
|
}
|
||||||
|
h.appendUnlogged(key)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H
|
// supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H
|
||||||
// §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its
|
// §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its
|
||||||
// pipeline-ended. newer is a pipeline just written waiting; a running pipeline is never touched, it runs to its end. It
|
// pipeline-ended. newer is a pipeline just written waiting; a running pipeline's movers leave it by SupersedeRunning
|
||||||
// answers the pipelines it ended.
|
// (H-A-223). It answers the pipelines it ended.
|
||||||
func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
|
func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
|
||||||
if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting {
|
if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
Reference in New Issue
Block a user