assigner: a reading step names the runs it read, and a superseded step leaves them (A5)

A step's end and its progress name per machine the composition run and its ask, so a
result is traced to the one ask that made it; a superseded or cancelled pipeline's
composition step leaves the runs it reads, the run going on for its other readers, and a
run left with no reader drops its ask and ends could not run.

Builds: design 47 G §723, §729 (mesh-wants' step-ended and step-progress shared); A-469n, A-469t, A-469z
This commit is contained in:
2026-11-17 11:38:10 +01:00
parent ba11ff58dd
commit d957a60286
7 changed files with 236 additions and 17 deletions
@@ -136,13 +136,22 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step
}
keys = append(keys, j.key)
}
return r.read(ctx, step.Module, keys, ends)
}
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
// runs it reads and their asks (§723, §729).
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
setShared(ctx, r.runs.sharedOf(keys, nil))
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
runs := r.runs.await(ctx, keys)
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
for _, key := range keys {
ends = append(ends, judgeRun(step.Module, runs[key]))
ends = append(ends, judgeRun(module, runs[key]))
}
return outcomeOf(ends)
out := outcomeOf(ends)
out.Shared = r.runs.sharedOf(keys, runs)
return out
}
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
@@ -155,12 +164,7 @@ func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.S
if err != nil {
return couldNotRun("the step's composition runs are not read: " + err.Error())
}
runs := r.runs.await(ctx, keys)
var ends []machineEnd
for _, key := range keys {
ends = append(ends, judgeRun(step.Module, runs[key]))
}
return outcomeOf(ends)
return r.read(ctx, step.Module, keys, nil)
}
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
@@ -360,6 +360,10 @@ func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
}
wg.Wait()
for i, o := range outs {
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
t.Errorf("step %d names the runs %+v", i, o.Shared)
}
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
t.Errorf("step %d came to %+v, want both machines read", i, o)
}
@@ -562,3 +562,70 @@ func unescape(part string) string {
}
return part
}
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
out := []meshwants.Shared{}
for _, key := range keys {
run, ok := runs[key]
if !ok {
if run, _, ok, _ = r.read(key); !ok {
continue
}
}
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
}
return out
}
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
// their asks' ids; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
func (r *compositionRuns) leave(pipeline string) ([]string, error) {
all, err := r.kv.keys()
if err != nil {
return nil, err
}
var ids []string
var errs []error
for _, key := range all {
if !isRunKey(key) {
continue
}
var left []string
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
left = nil
keep := c.Readers[:0:0]
for _, reader := range c.Readers {
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
left = append(left, reader.ID)
continue
}
keep = append(keep, reader)
}
c.Readers = keep
return len(left) > 0
})
if err != nil {
errs = append(errs, err)
continue
}
if !ok || len(left) == 0 {
continue
}
ids = append(ids, left...)
if run.Ended != nil || len(run.Readers) > 0 {
continue
}
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
if r.drop != nil {
if err := r.drop(run.Ask, why); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
errs = append(errs, err)
}
}
return ids, errors.Join(errs...)
}
@@ -23,6 +23,8 @@ type stepOutcome struct {
Outcome string
Said string
Warnings []string
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
Shared []meshwants.Shared
}
// compositionSteps takes the composition steps and says their ends.
@@ -39,6 +41,9 @@ type compositionSteps struct {
asker func() string
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
stops map[string]context.CancelFunc
}
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
@@ -70,10 +75,27 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
// composition runs it waits on (§723, §729).
type stepPhase struct {
mu sync.Mutex
phase string
mu sync.Mutex
phase string
shared []meshwants.Shared
}
// setShared names the runs the step ctx runs waits on, when it runs one.
func setShared(ctx context.Context, shared []meshwants.Shared) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.shared = shared
p.mu.Unlock()
}
}
func (p *stepPhase) waitsOn() []meshwants.Shared {
p.mu.Lock()
defer p.mu.Unlock()
return p.shared
}
type phaseKey struct{}
@@ -105,22 +127,51 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo
return true
}
// say1 runs one step with how and says its end.
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723). A step whose wait was ended by its
// pipeline's supersede or cancel (leave) says nothing: the runner ended it already.
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome) {
started := c.now().UTC()
ctx, stopWait := context.WithCancel(ctx)
c.mu.Lock()
if c.stops == nil {
c.stops = map[string]context.CancelFunc{}
}
c.stops[id] = stopWait
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.stops, id)
c.mu.Unlock()
stopWait()
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, phase)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
stop()
if ctx.Err() != nil {
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
return
}
ended := c.now().UTC()
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended, Shared: o.Shared}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
}
}
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
func (c *compositionSteps) leave(ids []string) {
c.mu.Lock()
defer c.mu.Unlock()
for _, id := range ids {
if stop := c.stops[id]; stop != nil {
stop()
}
}
}
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) {
@@ -140,7 +191,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -40,7 +42,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
@@ -67,7 +70,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
t.Fatalf("step-ended %+v", end)
}
}
@@ -105,6 +109,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
},
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
@@ -122,7 +127,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
said.mu.Lock()
p, ok := said.said[progress].(meshwants.StepProgressEvent)
said.mu.Unlock()
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
t.Fatalf("step-progress %+v", p)
}
close(release)
@@ -135,3 +141,71 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
t.Fatal("the ticker is not stopped when the step has ended")
}
}
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
// reader is left", so a later step starts a new run.
func TestASupersededStepLeavesItsRuns(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
step := func(pipeline string) meshwants.StepRun {
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
}
said := newSaidEvents()
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
if err != nil {
t.Error(err)
}
r.await(ctx, []string{j.key})
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
for _, p := range []string{"main.3.a", "main.3.b"} {
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
t.Fatal("not taken")
}
}
key := runKeyOf("main.3", "n1")
for {
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
break
}
<-r.wake()
}
ids, err := r.leave("main.3.a")
c.leave(ids)
run, _, _, _ := r.read(key)
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
}
ids, err = r.leave("main.3.b")
c.leave(ids)
run, _, _, _ = r.read(key)
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
}
deadline := time.After(5 * time.Second)
for {
c.mu.Lock()
n := len(c.stops)
c.mu.Unlock()
if n == 0 {
break
}
select {
case <-deadline:
t.Fatal("the steps that left still wait")
case <-time.After(time.Millisecond):
}
}
said.mu.Lock()
defer said.mu.Unlock()
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
t.Errorf("a step that left says its end: %v", said.said)
}
}
@@ -719,6 +719,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil {
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
}
// A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n).
if p.ends != nil {
left := func(pipeline string) error {
ids, err := runs.leave(pipeline)
steps.leave(ids)
return err
}
p.ends.left.Store(&left)
}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
@@ -153,6 +153,9 @@ type pipelineEnds struct {
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
runEnded func(group string) error
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
left atomic.Pointer[func(pipeline string) error]
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
@@ -285,6 +288,13 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
// §29, §64).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
if err := (*left)(p.Pipeline); err != nil {
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
_ = m.Nak()
return
}
}
switch {
case err != nil:
e.logf("%s is not taken: %v", m.Subject(), err)