From bbdd1741fa5baab1ce92b8611172dac44a380641 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 04:08:12 +0100 Subject: [PATCH] R13: a newer waiting pipeline supersedes the older one's movers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a pipeline is written waiting, each older waiting pipeline of its branch loses the movers the newer one also moves (read from the queue's waiting keys and each pipeline by its key); one left with no mover ends superseded, its waiting key deleted, saying wait-ended (how superseded) and its pipeline-ended. A running pipeline is never touched and runs to its end. The queue test's fixture is shared, its take-ins moving distinct modules. Not built: H-A-53's why of a mover whose dependency was superseded out (asked of the lead: no field keeps the movers that left). Builds: design 47 H §748; ADR 0682 §7; lead answer H-A-53 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/pipeline/queue_test.go | 101 ++++++++++++++---- .../internal/pipeline/run.go | 3 + .../internal/pipeline/start.go | 3 + .../internal/pipeline/supersede.go | 69 ++++++++++++ 4 files changed, 153 insertions(+), 23 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 5ca3f608..d964de37 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -65,21 +65,20 @@ func (m memQueue) Keys(prefix string) ([]string, error) { return out, nil } -// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a -// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started -// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places -// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline -// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed. -func TestThePipelinesWaitTheirTurn(t *testing.T) { +// queueFixture is a starter of pipelines-at-once 1 over a state pipelines and a state queue in memory, a hearer whose ends +// take their turn, and the verbs over both; said and waits record what they say. +func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *stepAsks, func(string) (Half, bool), Verbs) { reg, entries := placeFixture() + for _, m := range []string{"lib", "tool"} { + entries[m] = catalog.CatalogEntry{Module: m, Commit: "c1", Fp: "src2:" + m, Artifacts: entries["app"].Artifacts} + } mem, q := memStore{}, memQueue{} asks := &stepAsks{} - var said []string - var waits []seat.Wait + said, waits := &[]string{}, &[]seat.Wait{} say := func(subject string, body any) error { - said = append(said, subject[strings.Index(subject, ".event.")+7:]) + *said = append(*said, subject[strings.Index(subject, ".event.")+7:]) if w, ok := body.(seat.Wait); ok { - waits = append(waits, w) + *waits = append(*waits, w) } return nil } @@ -89,15 +88,28 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { Queue: &Queue{Store: q, AtOnce: 1}} view := func(string) (Half, bool) { return Half{Revision: 7, Entries: entries}, true } h := &Hearer{Store: mem, Name: "the-runner", Say: say, Now: now, Turn: func(key string) { _, _ = s.Turn(key, view) }} - v := Verbs{Hearer: h, Starter: s, Acts: &actLog{}} - take := func(source int64) { + s.Ended = func(p seat.Pipeline, steps []int) error { return h.EndedAtPlacing(p, steps) } + return s, mem, q, said, waits, asks, view, Verbs{Hearer: h, Starter: s, Acts: &actLog{}} +} + +// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a +// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started +// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places +// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline +// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed. +func TestThePipelinesWaitTheirTurn(t *testing.T) { + s, mem, q, said, waits, asks, view, v := queueFixture() + take := func(source int64, modules ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + for _, m := range modules { + ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) + } if n, err := s.TakenIn(ev, view); n != 1 || err != nil { t.Fatalf("the take-in of source %d started %d (%v)", source, n, err) } } - take(4) + take(4, "app") first := len(asks.ids) if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 { t.Fatalf("the first pipeline = %s seq %d with %d asks, want running and asked", p.Outcome, p.Seq, first) @@ -105,9 +117,9 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { if _, ok := q["running.main.4"]; !ok { t.Fatalf("the queue = %v, want running.main.4", q) } - said, waits = nil, nil - take(5) - take(6) + *said, *waits = nil, nil + take(5, "lib") + take(6, "tool") for i, key := range []string{"main.5", "main.6"} { p, _, _ := mem.Get(key) if p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != int64(i+1) || len(p.Steps) != 0 { @@ -120,12 +132,12 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { if _, ok := q["waiting.00000000000000000001"]; !ok || string(q["queue.next"].v) != "3" { t.Errorf("the queue = %v, want waiting.<1 in 20 digits> and queue.next 3", q) } - if !slices.Contains(said, "pipeline-started.main.5") || !slices.Contains(said, "wait-held.main.5") || len(waits) != 2 || - waits[0].Step != seat.WaitStepPipeline || waits[0].On != seat.WaitOnPipelinesAtOnce || waits[0].Pipeline != "main.5" { - t.Errorf("said %v, waits %+v; want pipeline-started and wait-held step pipeline on pipelines-at-once", said, waits) + if !slices.Contains(*said, "pipeline-started.main.5") || !slices.Contains(*said, "wait-held.main.5") || len(*waits) != 2 || + (*waits)[0].Step != seat.WaitStepPipeline || (*waits)[0].On != seat.WaitOnPipelinesAtOnce || (*waits)[0].Pipeline != "main.5" { + t.Errorf("said %v, waits %+v; want pipeline-started and wait-held step pipeline on pipelines-at-once", *said, *waits) } // R11/AC2: a cancel places the next, the lowest sequence. - said, waits = nil, nil + *said, *waits = nil, nil if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: "main.4", Why: "wrong"}); err != nil { t.Fatal(err) } @@ -141,8 +153,8 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { if gone || !runs || waits5 { t.Errorf("the queue after the cancel = %v", q) } - if !slices.Contains(said, "wait-ended.main.5") || len(waits) != 1 || waits[0].How != "ran" { - t.Errorf("said %v, waits %+v; want wait-ended of main.5", said, waits) + if !slices.Contains(*said, "wait-ended.main.5") || len(*waits) != 1 || (*waits)[0].How != "ran" { + t.Errorf("said %v, waits %+v; want wait-ended of main.5", *said, *waits) } // At the start: main.5's end was written while its key stayed, and a waiting key names a pipeline never written. p5, rev, _ := mem.Get("main.5") @@ -158,3 +170,46 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { t.Errorf("after the start main.6 = %s, queue %v", p.Outcome, q) } } + +// R13/AC1 (design 47 H §748, ADR 0682 §7): a newer waiting pipeline of the same branch takes the movers it also moves out of +// an older waiting one, which ends superseded, saying wait-ended and pipeline-ended, once no mover is left; a running +// pipeline moving the same module runs to its end. +func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) { + s, mem, q, said, waits, _, view, _ := queueFixture() + take := func(source int64, modules ...string) { + t.Helper() + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + for _, m := range modules { + ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) + } + if n, err := s.TakenIn(ev, view); n != 1 || err != nil { + t.Fatalf("the take-in of source %d started %d (%v)", source, n, err) + } + } + take(4, "app") + take(5, "app", "lib") + take(6, "app") + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting || len(p.Movers) != 1 || p.Movers[0].Module != "lib" { + t.Errorf("main.5 = %s moving %+v, want waiting with lib alone", p.Outcome, p.Movers) + } + if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || !Moves(p, "app") { + t.Errorf("the running main.4 = %s, want running to its end with app", p.Outcome) + } + *said, *waits = nil, nil + take(7, "lib") + p, _, _ := mem.Get("main.5") + if p.Outcome != seat.PipelineOutcomeSuperseded || p.Ended == nil { + t.Errorf("main.5 = %s, want superseded once no mover is left", p.Outcome) + } + if _, ok := q["waiting.00000000000000000001"]; ok { + t.Errorf("the queue still holds main.5's waiting key: %v", q) + } + if !slices.Contains(*said, "pipeline-ended.main.5") || !slices.ContainsFunc(*waits, func(w seat.Wait) bool { return w.Pipeline == "main.5" && w.How == "superseded" }) { + t.Errorf("said %v, waits %+v; want main.5's wait-ended superseded and its pipeline-ended", *said, *waits) + } + for _, key := range []string{"main.6", "main.7"} { + if p, _, _ := mem.Get(key); p.Outcome != seat.PipelineOutcomeWaiting { + t.Errorf("%s = %s, want waiting", key, p.Outcome) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index 987e44d9..f250c105 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -96,6 +96,9 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err } else { s.logf("the run %s waits its turn under pipelines-at-once, sequence %d", id, p.Seq) s.sayQueueWait(seat.EventWaitHeld, p, now, "") + if _, err := s.supersedeWaiting(p); err != nil { + s.logf("the older waiting pipelines of %s are not superseded by %s: %v", branch, key, err) + } } first := in.Members[0] last := first.Commit diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 63512835..9d3114c8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -454,6 +454,9 @@ func (s *Starter) Start(b Begin) (bool, error) { s.sayStartedPipeline(p, now) if !place { s.sayQueueWait(seat.EventWaitHeld, p, now, "") + if _, err := s.supersedeWaiting(p); err != nil { + s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err) + } return true, nil } s.ran(key) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index 5b283b8f..4c8e7a9c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -3,7 +3,10 @@ package pipeline import ( "errors" "fmt" + "slices" "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) // Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder @@ -44,3 +47,69 @@ func (h *Hearer) Taken(ask, node string) error { } func ptrTime(t time.Time) *time.Time { return &t } + +// supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H +// §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its +// pipeline-ended. newer is a pipeline just written waiting; a running pipeline is never touched, it runs to its end. It +// answers the pipelines it ended. +func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { + if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting { + return nil, nil + } + wkeys, pipelines, err := s.Queue.Waiting() + if err != nil { + return nil, err + } + var ended []string + for i, key := range pipelines { + if key == newer.Pipeline { + continue + } + done, err := s.supersedeOne(wkeys[i], key, newer) + if err != nil { + return ended, err + } + if done { + ended = append(ended, key) + } + } + return ended, nil +} + +// supersedeOne takes newer's movers out of the waiting pipeline key, keyed wkey in the queue; true when it ended superseded. +func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, error) { + for { + p, rev, err := s.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || p.Seq >= newer.Seq { + return false, nil + } + left := slices.DeleteFunc(slices.Clone(p.Movers), func(m seat.Mover) bool { return Moves(newer, m.Module) }) + if len(left) == len(p.Movers) { + return false, nil + } + gone := len(left) == 0 + now := s.now() + s.logf("%s: %d of its movers are moved by the newer %s and leave it", key, len(p.Movers)-len(left), newer.Pipeline) + p.Movers = left + if gone { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeSuperseded, "every mover is moved by the newer "+newer.Pipeline, &now + p.EndCount++ + } + if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return false, fmt.Errorf("the waiting pipeline %s is not written: %w", key, err) + } + if !gone { + return false, nil + } + if err := s.dropWaiting(wkey); err != nil { + return true, err + } + s.sayQueueWait(seat.EventWaitEnded, p, now, "superseded") + if s.Ended != nil { + return true, s.Ended(p, nil) + } + return true, nil + } +}