assigner: a reading step names the runs it read, and a superseded step leaves them (A5)
A step's end and its progress name per machine the composition run and its ask, so a result is traced to the one ask that made it; a superseded or cancelled pipeline's composition step leaves the runs it reads, the run going on for its other readers, and a run left with no reader drops its ask and ends could not run. Builds: design 47 G §723, §729 (mesh-wants' step-ended and step-progress shared); A-469n, A-469t, A-469z
This commit is contained in:
@@ -136,13 +136,22 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step
|
||||
}
|
||||
keys = append(keys, j.key)
|
||||
}
|
||||
return r.read(ctx, step.Module, keys, ends)
|
||||
}
|
||||
|
||||
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
|
||||
// runs it reads and their asks (§723, §729).
|
||||
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
|
||||
setShared(ctx, r.runs.sharedOf(keys, nil))
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
runs := r.runs.await(ctx, keys)
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
|
||||
for _, key := range keys {
|
||||
ends = append(ends, judgeRun(step.Module, runs[key]))
|
||||
ends = append(ends, judgeRun(module, runs[key]))
|
||||
}
|
||||
return outcomeOf(ends)
|
||||
out := outcomeOf(ends)
|
||||
out.Shared = r.runs.sharedOf(keys, runs)
|
||||
return out
|
||||
}
|
||||
|
||||
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
|
||||
@@ -155,12 +164,7 @@ func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.S
|
||||
if err != nil {
|
||||
return couldNotRun("the step's composition runs are not read: " + err.Error())
|
||||
}
|
||||
runs := r.runs.await(ctx, keys)
|
||||
var ends []machineEnd
|
||||
for _, key := range keys {
|
||||
ends = append(ends, judgeRun(step.Module, runs[key]))
|
||||
}
|
||||
return outcomeOf(ends)
|
||||
return r.read(ctx, step.Module, keys, nil)
|
||||
}
|
||||
|
||||
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
|
||||
|
||||
@@ -360,6 +360,10 @@ func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
|
||||
}
|
||||
wg.Wait()
|
||||
for i, o := range outs {
|
||||
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
|
||||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
|
||||
t.Errorf("step %d names the runs %+v", i, o.Shared)
|
||||
}
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
|
||||
t.Errorf("step %d came to %+v, want both machines read", i, o)
|
||||
}
|
||||
|
||||
@@ -562,3 +562,70 @@ func unescape(part string) string {
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
|
||||
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
|
||||
out := []meshwants.Shared{}
|
||||
for _, key := range keys {
|
||||
run, ok := runs[key]
|
||||
if !ok {
|
||||
if run, _, ok, _ = r.read(key); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
|
||||
// their asks' ids; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
|
||||
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
|
||||
func (r *compositionRuns) leave(pipeline string) ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
var errs []error
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
var left []string
|
||||
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
left = nil
|
||||
keep := c.Readers[:0:0]
|
||||
for _, reader := range c.Readers {
|
||||
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
|
||||
left = append(left, reader.ID)
|
||||
continue
|
||||
}
|
||||
keep = append(keep, reader)
|
||||
}
|
||||
c.Readers = keep
|
||||
return len(left) > 0
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if !ok || len(left) == 0 {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, left...)
|
||||
if run.Ended != nil || len(run.Readers) > 0 {
|
||||
continue
|
||||
}
|
||||
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, why); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return ids, errors.Join(errs...)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ type stepOutcome struct {
|
||||
Outcome string
|
||||
Said string
|
||||
Warnings []string
|
||||
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
|
||||
Shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// compositionSteps takes the composition steps and says their ends.
|
||||
@@ -39,6 +41,9 @@ type compositionSteps struct {
|
||||
asker func() string
|
||||
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
|
||||
ticks func(d time.Duration) (<-chan time.Time, func())
|
||||
mu sync.Mutex
|
||||
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
|
||||
stops map[string]context.CancelFunc
|
||||
}
|
||||
|
||||
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
|
||||
@@ -70,10 +75,27 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
|
||||
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
|
||||
const progressEvery = time.Minute
|
||||
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
|
||||
// composition runs it waits on (§723, §729).
|
||||
type stepPhase struct {
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// setShared names the runs the step ctx runs waits on, when it runs one.
|
||||
func setShared(ctx context.Context, shared []meshwants.Shared) {
|
||||
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
||||
p.mu.Lock()
|
||||
p.shared = shared
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *stepPhase) waitsOn() []meshwants.Shared {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.shared
|
||||
}
|
||||
|
||||
type phaseKey struct{}
|
||||
@@ -105,22 +127,51 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo
|
||||
return true
|
||||
}
|
||||
|
||||
// say1 runs one step with how and says its end.
|
||||
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723). A step whose wait was ended by its
|
||||
// pipeline's supersede or cancel (leave) says nothing: the runner ended it already.
|
||||
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome) {
|
||||
started := c.now().UTC()
|
||||
ctx, stopWait := context.WithCancel(ctx)
|
||||
c.mu.Lock()
|
||||
if c.stops == nil {
|
||||
c.stops = map[string]context.CancelFunc{}
|
||||
}
|
||||
c.stops[id] = stopWait
|
||||
c.mu.Unlock()
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
delete(c.stops, id)
|
||||
c.mu.Unlock()
|
||||
stopWait()
|
||||
}()
|
||||
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
|
||||
stop := c.sayProgress(id, phase)
|
||||
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
|
||||
return
|
||||
}
|
||||
ended := c.now().UTC()
|
||||
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
|
||||
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
|
||||
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
|
||||
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended, Shared: o.Shared}
|
||||
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
|
||||
c.logf("saying the end of the composition step %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
|
||||
func (c *compositionSteps) leave(ids []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
if stop := c.stops[id]; stop != nil {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
|
||||
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
|
||||
func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) {
|
||||
@@ -140,7 +191,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
|
||||
return
|
||||
case <-tick:
|
||||
at := c.now().UTC()
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()}
|
||||
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
|
||||
c.logf("saying the progress of the composition step %s: %v", id, err)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -40,7 +42,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
|
||||
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
|
||||
asked = run
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
|
||||
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
|
||||
}}
|
||||
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
|
||||
if c.take(context.Background(), "ask-0", other) {
|
||||
@@ -67,7 +70,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
|
||||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
|
||||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
|
||||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
|
||||
t.Fatalf("step-ended %+v", end)
|
||||
}
|
||||
}
|
||||
@@ -105,6 +109,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
},
|
||||
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
|
||||
<-release
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
@@ -122,7 +127,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
said.mu.Lock()
|
||||
p, ok := said.said[progress].(meshwants.StepProgressEvent)
|
||||
said.mu.Unlock()
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
|
||||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
|
||||
t.Fatalf("step-progress %+v", p)
|
||||
}
|
||||
close(release)
|
||||
@@ -135,3 +141,71 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
t.Fatal("the ticker is not stopped when the step has ended")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
|
||||
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
|
||||
// reader is left", so a later step starts a new run.
|
||||
func TestASupersededStepLeavesItsRuns(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
step := func(pipeline string) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
|
||||
}
|
||||
said := newSaidEvents()
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
|
||||
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
|
||||
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
r.await(ctx, []string{j.key})
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
|
||||
for _, p := range []string{"main.3.a", "main.3.b"} {
|
||||
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
|
||||
t.Fatal("not taken")
|
||||
}
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
for {
|
||||
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
|
||||
break
|
||||
}
|
||||
<-r.wake()
|
||||
}
|
||||
ids, err := r.leave("main.3.a")
|
||||
c.leave(ids)
|
||||
run, _, _, _ := r.read(key)
|
||||
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
|
||||
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
|
||||
}
|
||||
ids, err = r.leave("main.3.b")
|
||||
c.leave(ids)
|
||||
run, _, _, _ = r.read(key)
|
||||
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
|
||||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
|
||||
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
|
||||
}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
n := len(c.stops)
|
||||
c.mu.Unlock()
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("the steps that left still wait")
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
said.mu.Lock()
|
||||
defer said.mu.Unlock()
|
||||
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
|
||||
t.Errorf("a step that left says its end: %v", said.said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -719,6 +719,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil {
|
||||
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
// A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n).
|
||||
if p.ends != nil {
|
||||
left := func(pipeline string) error {
|
||||
ids, err := runs.leave(pipeline)
|
||||
steps.leave(ids)
|
||||
return err
|
||||
}
|
||||
p.ends.left.Store(&left)
|
||||
}
|
||||
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
|
||||
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -153,6 +153,9 @@ type pipelineEnds struct {
|
||||
firsts map[string]string
|
||||
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
|
||||
runEnded func(group string) error
|
||||
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
|
||||
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
|
||||
left atomic.Pointer[func(pipeline string) error]
|
||||
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
|
||||
onBranches func([]string)
|
||||
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
|
||||
@@ -285,6 +288,13 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
|
||||
// §29, §64).
|
||||
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
|
||||
p, err := mprc.PipelineEnded(m.Data())
|
||||
if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
|
||||
if err := (*left)(p.Pipeline); err != nil {
|
||||
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
e.logf("%s is not taken: %v", m.Subject(), err)
|
||||
|
||||
Reference in New Issue
Block a user