diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index 788e1758..d1873503 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -136,13 +136,22 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step } keys = append(keys, j.key) } + return r.read(ctx, step.Module, keys, ends) +} + +// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the +// runs it reads and their asks (§723, §729). +func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome { + setShared(ctx, r.runs.sharedOf(keys, nil)) setPhase(ctx, meshwants.StepProgressEventPhaseRun) runs := r.runs.await(ctx, keys) setPhase(ctx, meshwants.StepProgressEventPhaseReport) for _, key := range keys { - ends = append(ends, judgeRun(step.Module, runs[key])) + ends = append(ends, judgeRun(module, runs[key])) } - return outcomeOf(ends) + out := outcomeOf(ends) + out.Shared = r.runs.sharedOf(keys, runs) + return out } // resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended. @@ -155,12 +164,7 @@ func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.S if err != nil { return couldNotRun("the step's composition runs are not read: " + err.Error()) } - runs := r.runs.await(ctx, keys) - var ends []machineEnd - for _, key := range keys { - ends = append(ends, judgeRun(step.Module, runs[key])) - } - return outcomeOf(ends) + return r.read(ctx, step.Module, keys, nil) } // outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 943e5d1d..7024623e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -360,6 +360,10 @@ func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) { } wg.Wait() for i, o := range outs { + if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" || + o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask + t.Errorf("step %d names the runs %+v", i, o.Shared) + } if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") { t.Errorf("step %d came to %+v, want both machines read", i, o) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index 88208bfa..e3a223cc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -562,3 +562,70 @@ func unescape(part string) string { } return part } + +// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands. +func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared { + out := []meshwants.Shared{} + for _, key := range keys { + run, ok := runs[key] + if !ok { + if run, _, ok, _ = r.read(key); !ok { + continue + } + } + out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask}) + } + return out +} + +// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers +// their asks' ids; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z), +// so a later step of the candidate starts a new run. A run other readers read goes on for them. +func (r *compositionRuns) leave(pipeline string) ([]string, error) { + all, err := r.kv.keys() + if err != nil { + return nil, err + } + var ids []string + var errs []error + for _, key := range all { + if !isRunKey(key) { + continue + } + var left []string + run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool { + left = nil + keep := c.Readers[:0:0] + for _, reader := range c.Readers { + if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition { + left = append(left, reader.ID) + continue + } + keep = append(keep, reader) + } + c.Readers = keep + return len(left) > 0 + }) + if err != nil { + errs = append(errs, err) + continue + } + if !ok || len(left) == 0 { + continue + } + ids = append(ids, left...) + if run.Ended != nil || len(run.Readers) > 0 { + continue + } + why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled" + if r.drop != nil { + if err := r.drop(run.Ask, why); err != nil { + r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) + } + } + if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil { + errs = append(errs, err) + } + } + return ids, errors.Join(errs...) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 899be142..fa6e9208 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -23,6 +23,8 @@ type stepOutcome struct { Outcome string Said string Warnings []string + // Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729). + Shared []meshwants.Shared } // compositionSteps takes the composition steps and says their ends. @@ -39,6 +41,9 @@ type compositionSteps struct { asker func() string // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. ticks func(d time.Duration) (<-chan time.Time, func()) + mu sync.Mutex + // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave). + stops map[string]context.CancelFunc } // stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its @@ -70,10 +75,27 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string { // progressEvery is how often a running composition step says its progress (design 47 G §496, A-262). const progressEvery = time.Minute -// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context. +// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the +// composition runs it waits on (§723, §729). type stepPhase struct { - mu sync.Mutex - phase string + mu sync.Mutex + phase string + shared []meshwants.Shared +} + +// setShared names the runs the step ctx runs waits on, when it runs one. +func setShared(ctx context.Context, shared []meshwants.Shared) { + if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok { + p.mu.Lock() + p.shared = shared + p.mu.Unlock() + } +} + +func (p *stepPhase) waitsOn() []meshwants.Shared { + p.mu.Lock() + defer p.mu.Unlock() + return p.shared } type phaseKey struct{} @@ -105,22 +127,51 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo return true } -// say1 runs one step with how and says its end. +// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723). A step whose wait was ended by its +// pipeline's supersede or cancel (leave) says nothing: the runner ended it already. func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome) { started := c.now().UTC() + ctx, stopWait := context.WithCancel(ctx) + c.mu.Lock() + if c.stops == nil { + c.stops = map[string]context.CancelFunc{} + } + c.stops[id] = stopWait + c.mu.Unlock() + defer func() { + c.mu.Lock() + delete(c.stops, id) + c.mu.Unlock() + stopWait() + }() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} stop := c.sayProgress(id, phase) o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) stop() + if ctx.Err() != nil { + c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx)) + return + } ended := c.now().UTC() end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module, Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings, - Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended} + Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended, Shared: o.Shared} if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil { c.logf("saying the end of the composition step %s: %v", id, err) } } +// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs). +func (c *compositionSteps) leave(ids []string) { + c.mu.Lock() + defer c.mu.Unlock() + for _, id := range ids { + if stop := c.stops[id]; stop != nil { + stop() + } + } +} + // sayProgress says the step's progress on the seat's step-progress. once a minute until the returned stop is called (design // 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat. func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) { @@ -140,7 +191,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func() return case <-tick: at := c.now().UTC() - p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at} + p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()} if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil { c.logf("saying the progress of the composition step %s: %v", id, err) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index 7467700d..7057b980 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "git.novox.be/novox/mesh-sdk/go/seatdef" + "slices" + "strings" "sync" "testing" "time" @@ -40,7 +42,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf, run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome { asked = run - return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}} + return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}, + Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}} }} other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"}) if c.take(context.Background(), "ask-0", other) { @@ -67,7 +70,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition || end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" || end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 || - !end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil { + !end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil || + len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729 t.Fatalf("step-ended %+v", end) } } @@ -105,6 +109,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { }, run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome { setPhase(ctx, meshwants.StepProgressEventPhaseRun) + setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}) <-release return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} }} @@ -122,7 +127,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { said.mu.Lock() p, ok := said.said[progress].(meshwants.StepProgressEvent) said.mu.Unlock() - if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) { + if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) || + len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723) t.Fatalf("step-progress %+v", p) } close(release) @@ -135,3 +141,71 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { t.Fatal("the ticker is not stopped when the step has ended") } } + +// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end; +// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no +// reader is left", so a later step starts a new run. +func TestASupersededStepLeavesItsRuns(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf) + var drops []string + r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil } + step := func(pipeline string) meshwants.StepRun { + return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3} + } + said := newSaidEvents() + c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf, + run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome { + j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id) + if err != nil { + t.Error(err) + } + r.await(ctx, []string{j.key}) + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + }} + body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw } + for _, p := range []string{"main.3.a", "main.3.b"} { + if !c.take(context.Background(), "ask-"+p, body(step(p))) { + t.Fatal("not taken") + } + } + key := runKeyOf("main.3", "n1") + for { + if run, _, _, _ := r.read(key); len(run.Readers) == 2 { + break + } + <-r.wake() + } + ids, err := r.leave("main.3.a") + c.leave(ids) + run, _, _, _ := r.read(key) + if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 { + t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops) + } + ids, err = r.leave("main.3.b") + c.leave(ids) + run, _, _, _ = r.read(key) + if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 || + !strings.HasPrefix(drops[0], run.Ask+": no reader is left") { + t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops) + } + deadline := time.After(5 * time.Second) + for { + c.mu.Lock() + n := len(c.stops) + c.mu.Unlock() + if n == 0 { + break + } + select { + case <-deadline: + t.Fatal("the steps that left still wait") + case <-time.After(time.Millisecond): + } + } + said.mu.Lock() + defer said.mu.Unlock() + if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok { + t.Errorf("a step that left says its end: %v", said.said) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 337c5bba..72d2da70 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -719,6 +719,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil { return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err) } + // A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n). + if p.ends != nil { + left := func(pipeline string) error { + ids, err := runs.leave(pipeline) + steps.leave(ids) + return err + } + p.ends.left.Store(&left) + } if err := serveCompositionSteps(ctx, nc, steps); err != nil { logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err) } else { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 03c481d2..5baf09ba 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -153,6 +153,9 @@ type pipelineEnds struct { firsts map[string]string // runEnded is told a run on request's group-ended, its group `run-`, to drop the run's candidate (A-235); nil drops nothing. runEnded func(group string) error + // left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read + // (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts. + left atomic.Pointer[func(pipeline string) error] // onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195). onBranches func([]string) // trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none. @@ -285,6 +288,13 @@ func (e *pipelineEnds) branchOf(module, placed string) string { // §29, §64). func (e *pipelineEnds) hearPipeline(m jetstream.Msg) { p, err := mprc.PipelineEnded(m.Data()) + if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) { + if err := (*left)(p.Pipeline); err != nil { + e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err) + _ = m.Nak() + return + } + } switch { case err != nil: e.logf("%s is not taken: %v", m.Subject(), err)