diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 85891217..1597d3e7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -160,6 +160,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps + starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 1b2e0911..68dd3f35 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -162,6 +162,9 @@ type Starter struct { // Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end // (Hearer.EndedAtPlacing); nil says it in the journal alone. Ended func(p seat.Pipeline, steps []int) error + // Supersede takes out of every older running pipeline of newer's branch each mover whose fingerprint newer moved + // (H-A-223; Hearer.SupersedeRunning); nil leaves running pipelines as they are. + Supersede func(newer seat.Pipeline) // Fills refuses a start while a store of the runner is full (§366); nil refuses none. Fills *Fills // Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery. @@ -538,6 +541,10 @@ func (s *Starter) Start(b Begin) (bool, error) { // A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its // wait on its turn. s.sayStartedPipeline(p, now) + // A mover whose fingerprint this revision moved leaves every older running pipeline of the branch (H-A-223). + if s.Supersede != nil { + s.Supersede(p) + } if !place { s.sayQueueWait(seat.EventWaitHeld, p, now, "") if _, err := s.supersedeWaiting(p); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 40524451..78a4d3f8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strings" "time" @@ -611,9 +612,16 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s } // A mover with no step placed waits on its registration or a publish: it holds the end only while every placed mover's // required steps passed, since once one failed or could not run the pipeline can no longer pass (H-A-163, H-A-171). + // A mover that left for a newer pipeline (H-A-223) holds nothing and fails nothing: its steps are not judged, and a + // pipeline every mover left ends superseded. + if allLeft(*p) { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeSuperseded, "every mover left it for a newer pipeline (H-A-223)", &now + p.EndCount++ + return true + } var waiting []string for _, m := range p.Movers { - if !placed(*p, m.Module) { + if !placed(*p, m.Module) && !slices.Contains(p.Superseded, m.Module) { waiting = append(waiting, m.Module) } } @@ -621,6 +629,8 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s var notAsked []int for i, s := range p.Steps { switch { + case slices.Contains(p.Superseded, s.Module): + continue case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting: notAsked = append(notAsked, i) continue diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go index 5dc67bc0..104c792b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -1,7 +1,9 @@ package pipeline import ( + "encoding/json" "errors" + "fmt" "strings" "testing" "time" @@ -377,3 +379,105 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) { t.Errorf("recovered: said %v", said) } } + +// A mover whose fingerprint a newer revision moved leaves a running pipeline (H-A-223, H-A-163): its open steps end +// superseded, never failed, the other mover's go on and the pipeline ends by them alone; one every mover left ends +// superseded. +func TestAMovedMoverLeavesARunningPipeline(t *testing.T) { + t0 := time.Date(2026, 11, 21, 1, 0, 0, 0, time.UTC) + running := func() seat.Pipeline { + return seat.Pipeline{V: 1, Pipeline: "main.54", Branch: "main", Source: 54, Revision: 54, Outcome: seat.PipelineOutcomeRunning, Started: &t0, + Movers: []seat.Mover{{Module: "a", Commit: "c1", Fingerprint: "src2:a1"}, {Module: "b", Commit: "c1", Fingerprint: "src2:b1"}}, + Steps: []seat.StepResult{ + {Module: "a", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x1", Started: &t0}, + {Module: "b", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x2", Started: &t0}, + {Module: "b", Step: seat.StepResultStepLint, Artifact: "b:program", Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x3", Started: &t0, + Seat: nodebuildgo.Seat, Node: "n2"}}} + } + rows := []struct { + name string + newer []seat.Mover + want string + }{ + {"a newer revision moving b: b's steps end superseded, a's go on and end the pipeline passed alone", + []seat.Mover{{Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}, "running; a build running; b build superseded; b lint superseded | passed"}, + {"a newer revision moving every mover ends the pipeline superseded", + []seat.Mover{{Module: "a", Commit: "c2", Fingerprint: "src2:a2"}, {Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}, + "superseded; a build superseded; b build superseded; b lint superseded | superseded"}, + {"a newer revision at the same fingerprint takes nothing", []seat.Mover{{Module: "b", Commit: "c1", Fingerprint: "src2:b1"}}, + "running; a build running; b build running; b lint running | running"}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + store := memStore{} + store["main.54"] = struct { + p seat.Pipeline + rev uint64 + }{running(), 1} + var said, dropped []string + h := &Hearer{Store: store, Say: func(subject string, _ any) error { said = append(said, subject); return nil }, Now: func() time.Time { return t0 }, + DropRun: func(buildSeat, node, ask, why string) error { + dropped = append(dropped, buildSeat+" "+node+" "+ask+" "+fmt.Sprint(strings.Contains(why, "main.57"))) + return nil + }} + h.SupersedeRunning(seat.Pipeline{Pipeline: "main.57", Branch: "main", Revision: 57, Movers: r.newer}) + // b's lint ask on its build node is dropped naming the newer pipeline; the build ask on mesh-builds, which + // serves no drop, is not (H-A-223). + wantDropped := "" + if r.newer[0].Fingerprint != "src2:b1" { + wantDropped = "node-build-go n2 x3 true" + } + if got := strings.Join(dropped, ","); got != wantDropped { + t.Errorf("dropped %q; want %q", got, wantDropped) + } + p, _, _ := store.Get("main.54") + got := []string{p.Outcome} + for _, s := range p.Steps { + got = append(got, s.Module+" "+s.Step+" "+s.Outcome) + if s.Outcome == seat.StepResultOutcomeSuperseded && !strings.Contains(s.Why, "main.57") { + t.Errorf("the superseded step's why %q names no newer pipeline", s.Why) + } + } + // The mover left holds nothing: a's build passing ends the pipeline by a alone. + if p.Outcome == seat.PipelineOutcomeRunning { + p.Steps[0].Outcome, p.Steps[0].Ended = seat.StepResultOutcomePassed, &t0 + endPipeline(&p, t0, nil) + } + if s := strings.Join(got, "; ") + " | " + p.Outcome; s != r.want { + t.Errorf("got %s\nwant %s", s, r.want) + } + }) + } + // The live path: a newer pipeline started through Start, a take-in's or a release revision's alike, takes the moved + // mover out of the older running pipeline over the same store (H-A-223, H-A-213). + starts := []struct { + name string + begin Begin + }{ + {"a take-in", Begin{Branch: "main", Source: 55, Revision: 55, Call: "0123456789abcdef", Movers: []seat.Mover{{Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}}}, + {"a release revision", Begin{Branch: "main", Source: 54, Revision: 57, Call: "release of mesh-sdk 0.1.26", Movers: []seat.Mover{{Module: "b", Commit: "c1", Fingerprint: "src2:b2"}}}}, + } + for _, r := range starts { + t.Run("started by "+r.name, func(t *testing.T) { + store := &MoverStore{Raw: newMemRaw()} + raw, _ := json.Marshal(running()) + if err := store.Create("main.54", raw); err != nil { + t.Fatal(err) + } + h := &Hearer{Store: store, Say: func(string, any) error { return nil }, Now: func() time.Time { return t0 }} + s := &Starter{Pipelines: store, Store: store, Say: func(string, any) error { return nil }, Now: func() time.Time { return t0 }, + Supersede: func(newer seat.Pipeline) { h.SupersedeRunning(newer) }} + if ok, err := s.Start(r.begin); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := store.Get("main.54") + var got []string + for _, st := range p.Steps { + got = append(got, st.Module+" "+st.Step+" "+st.Outcome) + } + if s := p.Outcome + "; " + strings.Join(got, "; "); s != "running; a build running; b build superseded; b lint superseded" { + t.Errorf("the older pipeline after %s = %s; want b left it", r.name, s) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index 5947724b..6d77cbda 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -9,6 +9,7 @@ import ( catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) // Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder @@ -50,10 +51,120 @@ func (h *Hearer) Taken(ask, node string) error { func ptrTime(t time.Time) *time.Time { return &t } +// allLeft says whether every mover of p left it for a newer pipeline (H-A-223). +func allLeft(p seat.Pipeline) bool { + if len(p.Movers) == 0 { + return false + } + for _, m := range p.Movers { + if !slices.Contains(p.Superseded, m.Module) { + return false + } + } + return true +} + +// SupersedeRunning takes out of every older running pipeline of newer's branch, read by that branch's keys alone, each mover whose fingerprint newer moved +// (H-A-223): its open steps end superseded, naming newer, never failed; a step reading a test run leaves it, as a cancel's +// does; the mover is kept in the pipeline's superseded, so the end judgement (H-A-163) counts it as left and newer carries +// it. A pipeline then judged ends as at a step's end, by its other movers alone; one every mover left ends superseded. +// A run on request's pipeline is never touched. It answers the pipelines it touched. +func (h *Hearer) SupersedeRunning(newer seat.Pipeline) []string { + pkeys, err := branchKeys(h.Store, newer.Branch) + if err != nil { + h.logf("the running pipelines of %s are not listed, so none is superseded by %s: %v", newer.Branch, newer.Pipeline, err) + return nil + } + var touched []string + for _, key := range pkeys { + if key == newer.Pipeline { + continue + } + if h.supersedeRunningOne(key, newer) { + touched = append(touched, key) + } + } + return touched +} + +func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool { + for { + p, rev, err := h.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeRunning || p.Branch != newer.Branch || p.Run != "" || newer.Run != "" || + p.Revision >= newer.Revision { + return false + } + var moved []string + for _, m := range p.Movers { + if slices.Contains(p.Superseded, m.Module) { + continue + } + if nm, ok := MoverOf(newer, m.Module); ok && nm.Fingerprint != m.Fingerprint { + moved = append(moved, m.Module) + } + } + if len(moved) == 0 { + return false + } + now := h.now() + why := "superseded: the newer " + newer.Pipeline + " moved its fingerprint, and carries it (H-A-223)" + var ended []int + var leave, drop []seat.StepResult + for i := range p.Steps { + st := &p.Steps[i] + if !slices.Contains(moved, st.Module) || (!open(*st) && st.Outcome != seat.StepResultOutcomeWaiting) { + continue + } + switch { + case sharedRun(*st) != "" && st.Seat != wants.Seat: + leave = append(leave, *st) + case st.Ask != "" && BoundSeat(st.Seat): + drop = append(drop, *st) // its own ask on a build node: the seat's drop cancels it + } + st.Outcome, st.Why, st.Ended = seat.StepResultOutcomeSuperseded, why, &now + p.Unlogged = append(p.Unlogged, endEntry(key, *st, now)) + ended = append(ended, i) + } + p.Superseded = append(p.Superseded, moved...) + done := h.endPipeline(&p, now) + if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the running pipeline %s is not written, so its movers moved by %s stay: %v", key, newer.Pipeline, err) + return false + } + h.logf("%s: %s moved by the newer %s leave it, %d of its open steps ended superseded (H-A-223)", key, strings.Join(moved, ", "), newer.Pipeline, len(ended)) + for _, i := range ended { + if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil { + h.logf("step-ended of %s's %s %s, superseded, is not said: %v", key, p.Steps[i].Module, p.Steps[i].Step, err) + } + } + for _, st := range leave { + h.leaveRun(key, st, "the mover "+st.Module+" left "+key+" for the newer "+newer.Pipeline) + } + // Its running asks on the build nodes are cancelled through the seat's drop (H-A-223). The seat mesh-builds serves no + // drop, so a build ask there runs to its end, which the runner ignores: the step is no longer open. + for _, st := range drop { + if h.DropRun == nil { + h.logf("%s: the ask %s of %s's %s is not dropped: the runner has no drop", key, st.Ask, st.Module, st.Step) + continue + } + if err := h.DropRun(st.Seat, st.Node, st.Ask, why); err != nil { + h.logf("%s: the drop of the ask %s of %s's %s is not asked of %s: %v", key, st.Ask, st.Module, st.Step, st.Seat, err) + } + } + if err := h.said(p, nil, done); err != nil { + h.logf("%v", err) + } + h.appendUnlogged(key) + return true + } +} + // supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H // §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its -// pipeline-ended. newer is a pipeline just written waiting; a running pipeline is never touched, it runs to its end. It -// answers the pipelines it ended. +// pipeline-ended. newer is a pipeline just written waiting; a running pipeline's movers leave it by SupersedeRunning +// (H-A-223). It answers the pipelines it ended. func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting { return nil, nil