Design 47 H1: a moved mover leaves a running pipeline too (H-A-223)

A release re-pinned node-tools while main.54 still built it at its old
fingerprint; the build failed and failed the pipeline for its other 119
movers, because supersession took movers out of waiting pipelines only.
Every start of a newer revision's pipeline now also walks the older running
pipelines of its branch: each mover whose fingerprint it moved leaves, its
open steps ending superseded with a why naming the newer pipeline, a step
reading a test run leaving it as a cancel's does, and the mover kept in the
pipeline's superseded. The end judgement counts a left mover's steps as
neither holding nor failed, so the pipeline ends by its other movers alone;
one every mover left ends superseded. A run on request's pipeline is never
touched.

Builds: design 47 H H-A-223, H-A-163, H-A-213
This commit is contained in:
2026-11-21 01:13:57 +01:00
parent 8802f16019
commit 06a52a10e0
5 changed files with 174 additions and 3 deletions
@@ -160,6 +160,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
hearer.AskSteps = starter.AskSteps hearer.AskSteps = starter.AskSteps
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
hearer.AskPinHeld = func(key, module string, pins map[string]string) { hearer.AskPinHeld = func(key, module string, pins map[string]string) {
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
@@ -162,6 +162,9 @@ type Starter struct {
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end // Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
// (Hearer.EndedAtPlacing); nil says it in the journal alone. // (Hearer.EndedAtPlacing); nil says it in the journal alone.
Ended func(p seat.Pipeline, steps []int) error Ended func(p seat.Pipeline, steps []int) error
// Supersede takes out of every older running pipeline of newer's branch each mover whose fingerprint newer moved
// (H-A-223; Hearer.SupersedeRunning); nil leaves running pipelines as they are.
Supersede func(newer seat.Pipeline)
// Fills refuses a start while a store of the runner is full (§366); nil refuses none. // Fills refuses a start while a store of the runner is full (§366); nil refuses none.
Fills *Fills Fills *Fills
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery. // Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
@@ -538,6 +541,10 @@ func (s *Starter) Start(b Begin) (bool, error) {
// A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its // A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its
// wait on its turn. // wait on its turn.
s.sayStartedPipeline(p, now) s.sayStartedPipeline(p, now)
// A mover whose fingerprint this revision moved leaves every older running pipeline of the branch (H-A-223).
if s.Supersede != nil {
s.Supersede(p)
}
if !place { if !place {
s.sayQueueWait(seat.EventWaitHeld, p, now, "") s.sayQueueWait(seat.EventWaitHeld, p, now, "")
if _, err := s.supersedeWaiting(p); err != nil { if _, err := s.supersedeWaiting(p); err != nil {
@@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"slices"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -611,9 +612,16 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
} }
// A mover with no step placed waits on its registration or a publish: it holds the end only while every placed mover's // A mover with no step placed waits on its registration or a publish: it holds the end only while every placed mover's
// required steps passed, since once one failed or could not run the pipeline can no longer pass (H-A-163, H-A-171). // required steps passed, since once one failed or could not run the pipeline can no longer pass (H-A-163, H-A-171).
// A mover that left for a newer pipeline (H-A-223) holds nothing and fails nothing: its steps are not judged, and a
// pipeline every mover left ends superseded.
if allLeft(*p) {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeSuperseded, "every mover left it for a newer pipeline (H-A-223)", &now
p.EndCount++
return true
}
var waiting []string var waiting []string
for _, m := range p.Movers { for _, m := range p.Movers {
if !placed(*p, m.Module) { if !placed(*p, m.Module) && !slices.Contains(p.Superseded, m.Module) {
waiting = append(waiting, m.Module) waiting = append(waiting, m.Module)
} }
} }
@@ -621,6 +629,8 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
var notAsked []int var notAsked []int
for i, s := range p.Steps { for i, s := range p.Steps {
switch { switch {
case slices.Contains(p.Superseded, s.Module):
continue
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting: case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
notAsked = append(notAsked, i) notAsked = append(notAsked, i)
continue continue
@@ -377,3 +377,59 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
t.Errorf("recovered: said %v", said) t.Errorf("recovered: said %v", said)
} }
} }
// A mover whose fingerprint a newer revision moved leaves a running pipeline (H-A-223, H-A-163): its open steps end
// superseded, never failed, the other mover's go on and the pipeline ends by them alone; one every mover left ends
// superseded.
func TestAMovedMoverLeavesARunningPipeline(t *testing.T) {
t0 := time.Date(2026, 11, 21, 1, 0, 0, 0, time.UTC)
running := func() seat.Pipeline {
return seat.Pipeline{V: 1, Pipeline: "main.54", Branch: "main", Source: 54, Revision: 54, Outcome: seat.PipelineOutcomeRunning, Started: &t0,
Movers: []seat.Mover{{Module: "a", Commit: "c1", Fingerprint: "src2:a1"}, {Module: "b", Commit: "c1", Fingerprint: "src2:b1"}},
Steps: []seat.StepResult{
{Module: "a", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x1", Started: &t0},
{Module: "b", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x2", Started: &t0},
{Module: "b", Step: seat.StepResultStepLint, Artifact: "b:program", Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x3", Started: &t0}}}
}
rows := []struct {
name string
newer []seat.Mover
want string
}{
{"a newer revision moving b: b's steps end superseded, a's go on and end the pipeline passed alone",
[]seat.Mover{{Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}, "running; a build running; b build superseded; b lint superseded | passed"},
{"a newer revision moving every mover ends the pipeline superseded",
[]seat.Mover{{Module: "a", Commit: "c2", Fingerprint: "src2:a2"}, {Module: "b", Commit: "c2", Fingerprint: "src2:b2"}},
"superseded; a build superseded; b build superseded; b lint superseded | superseded"},
{"a newer revision at the same fingerprint takes nothing", []seat.Mover{{Module: "b", Commit: "c1", Fingerprint: "src2:b1"}},
"running; a build running; b build running; b lint running | running"},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
store := memStore{}
store["main.54"] = struct {
p seat.Pipeline
rev uint64
}{running(), 1}
var said []string
h := &Hearer{Store: store, Say: func(subject string, _ any) error { said = append(said, subject); return nil }, Now: func() time.Time { return t0 }}
h.SupersedeRunning(seat.Pipeline{Pipeline: "main.57", Branch: "main", Revision: 57, Movers: r.newer})
p, _, _ := store.Get("main.54")
got := []string{p.Outcome}
for _, s := range p.Steps {
got = append(got, s.Module+" "+s.Step+" "+s.Outcome)
if s.Outcome == seat.StepResultOutcomeSuperseded && !strings.Contains(s.Why, "main.57") {
t.Errorf("the superseded step's why %q names no newer pipeline", s.Why)
}
}
// The mover left holds nothing: a's build passing ends the pipeline by a alone.
if p.Outcome == seat.PipelineOutcomeRunning {
p.Steps[0].Outcome, p.Steps[0].Ended = seat.StepResultOutcomePassed, &t0
endPipeline(&p, t0, nil)
}
if s := strings.Join(got, "; ") + " | " + p.Outcome; s != r.want {
t.Errorf("got %s\nwant %s", s, r.want)
}
})
}
}
@@ -9,6 +9,7 @@ import (
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
) )
// Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder // Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder
@@ -50,10 +51,106 @@ func (h *Hearer) Taken(ask, node string) error {
func ptrTime(t time.Time) *time.Time { return &t } func ptrTime(t time.Time) *time.Time { return &t }
// allLeft says whether every mover of p left it for a newer pipeline (H-A-223).
func allLeft(p seat.Pipeline) bool {
if len(p.Movers) == 0 {
return false
}
for _, m := range p.Movers {
if !slices.Contains(p.Superseded, m.Module) {
return false
}
}
return true
}
// SupersedeRunning takes out of every older running pipeline of newer's branch each mover whose fingerprint newer moved
// (H-A-223): its open steps end superseded, naming newer, never failed; a step reading a test run leaves it, as a cancel's
// does; the mover is kept in the pipeline's superseded, so the end judgement (H-A-163) counts it as left and newer carries
// it. A pipeline then judged ends as at a step's end, by its other movers alone; one every mover left ends superseded.
// A run on request's pipeline is never touched. It answers the pipelines it touched.
func (h *Hearer) SupersedeRunning(newer seat.Pipeline) []string {
pkeys, err := allKeys(h.Store)
if err != nil {
h.logf("the running pipelines are not listed, so none is superseded by %s: %v", newer.Pipeline, err)
return nil
}
var touched []string
for _, key := range pkeys {
if key == newer.Pipeline {
continue
}
if h.supersedeRunningOne(key, newer) {
touched = append(touched, key)
}
}
return touched
}
func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool {
for {
p, rev, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || p.Branch != newer.Branch || p.Run != "" || newer.Run != "" ||
p.Revision >= newer.Revision {
return false
}
var moved []string
for _, m := range p.Movers {
if slices.Contains(p.Superseded, m.Module) {
continue
}
if nm, ok := MoverOf(newer, m.Module); ok && nm.Fingerprint != m.Fingerprint {
moved = append(moved, m.Module)
}
}
if len(moved) == 0 {
return false
}
now := h.now()
why := "superseded: the newer " + newer.Pipeline + " moved its fingerprint, and carries it (H-A-223)"
var ended []int
var leave []seat.StepResult
for i := range p.Steps {
st := &p.Steps[i]
if !slices.Contains(moved, st.Module) || (!open(*st) && st.Outcome != seat.StepResultOutcomeWaiting) {
continue
}
if sharedRun(*st) != "" && st.Seat != wants.Seat {
leave = append(leave, *st)
}
st.Outcome, st.Why, st.Ended = seat.StepResultOutcomeSuperseded, why, &now
p.Unlogged = append(p.Unlogged, endEntry(key, *st, now))
ended = append(ended, i)
}
p.Superseded = append(p.Superseded, moved...)
done := h.endPipeline(&p, now)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the running pipeline %s is not written, so its movers moved by %s stay: %v", key, newer.Pipeline, err)
return false
}
h.logf("%s: %s moved by the newer %s leave it, %d of its open steps ended superseded (H-A-223)", key, strings.Join(moved, ", "), newer.Pipeline, len(ended))
for _, i := range ended {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil {
h.logf("step-ended of %s's %s %s, superseded, is not said: %v", key, p.Steps[i].Module, p.Steps[i].Step, err)
}
}
for _, st := range leave {
h.leaveRun(key, st, "the mover "+st.Module+" left "+key+" for the newer "+newer.Pipeline)
}
if err := h.said(p, nil, done); err != nil {
h.logf("%v", err)
}
h.appendUnlogged(key)
return true
}
}
// supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H // supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H
// §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its // §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its
// pipeline-ended. newer is a pipeline just written waiting; a running pipeline is never touched, it runs to its end. It // pipeline-ended. newer is a pipeline just written waiting; a running pipeline's movers leave it by SupersedeRunning
// answers the pipelines it ended. // (H-A-223). It answers the pipelines it ended.
func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting { if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting {
return nil, nil return nil, nil