The assigner takes the seat mesh-wants' step validate beside the composition step. Every mover's validate step of one take-in reads one run, the key <pipeline>.validate of composition-runs with no machine, taken up at the start like the composer's runs: a cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from the revision's entry of the module holding mesh-catalog runs on the Go build seat, its input kept and served on mesh-wants' input, over the revision's manifests and the branch's previous revision's. The step ends failed cycle, failed manifest-refused naming every problem the revision causes, or passed. Builds: design 47 H §745; H-A-34, H-A-51, H-A-108 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
264 lines
9.7 KiB
Go
264 lines
9.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
)
|
|
|
|
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
|
|
// composition step of one module's pipeline that the runner asks, runs it, and says its end as the seat's event
|
|
// step-ended.<node>, the ask's id as the event's id and the event's `id`, which the runner holds the end to.
|
|
|
|
// stepOutcome is what one composition step came to.
|
|
type stepOutcome struct {
|
|
Outcome string
|
|
Said string
|
|
Warnings []string
|
|
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
|
|
Shared []meshwants.Shared
|
|
}
|
|
|
|
// compositionSteps takes the composition steps and says their ends.
|
|
type compositionSteps struct {
|
|
node string
|
|
// say says one event of the seat mesh-wants (wantsPublisher).
|
|
say publisher
|
|
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
|
|
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
|
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
|
// validate runs one validate step and resumeValidate judges one taken up at the start (validatestep.go, design 47 H §745);
|
|
// nil takes no validate step.
|
|
validate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
|
resumeValidate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
|
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
|
|
// writes none.
|
|
ended func(id string, run meshwants.StepRun, at time.Time)
|
|
now func() time.Time
|
|
logf func(string, ...any)
|
|
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
|
|
ticks func(d time.Duration) (<-chan time.Time, func())
|
|
mu sync.Mutex
|
|
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
|
|
stops map[string]context.CancelFunc
|
|
}
|
|
|
|
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
|
|
const progressEvery = time.Minute
|
|
|
|
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
|
|
// composition runs it waits on (§723, §729).
|
|
type stepPhase struct {
|
|
mu sync.Mutex
|
|
phase string
|
|
shared []meshwants.Shared
|
|
}
|
|
|
|
// setShared names the runs the step ctx runs waits on, when it runs one.
|
|
func setShared(ctx context.Context, shared []meshwants.Shared) {
|
|
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
|
p.mu.Lock()
|
|
p.shared = shared
|
|
p.mu.Unlock()
|
|
}
|
|
}
|
|
|
|
func (p *stepPhase) waitsOn() []meshwants.Shared {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.shared
|
|
}
|
|
|
|
type phaseKey struct{}
|
|
|
|
// setPhase sets the phase of the step ctx runs, when it runs one.
|
|
func setPhase(ctx context.Context, phase string) {
|
|
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
|
p.mu.Lock()
|
|
p.phase = phase
|
|
p.mu.Unlock()
|
|
}
|
|
}
|
|
|
|
func (p *stepPhase) get() string {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.phase
|
|
}
|
|
|
|
// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition or validate step, or an
|
|
// ask with no id, is left with a log line; one taken is run in its own goroutine and its end said.
|
|
func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool {
|
|
var run meshwants.StepRun
|
|
err := json.Unmarshal(data, &run)
|
|
how := c.howOf(run.Step, false)
|
|
if err != nil || id == "" || how == nil {
|
|
c.logf("a step ask that is no composition or validate step with an id is left: %v", err)
|
|
return false
|
|
}
|
|
go c.say1(ctx, id, run, how, "")
|
|
return true
|
|
}
|
|
|
|
// howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another.
|
|
func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome {
|
|
switch {
|
|
case step == meshwants.StepRunStepComposition && resumed:
|
|
return c.resume
|
|
case step == meshwants.StepRunStepComposition:
|
|
return c.run
|
|
case step == meshwants.StepRunStepValidate && resumed:
|
|
return c.resumeValidate
|
|
case step == meshwants.StepRunStepValidate:
|
|
return c.validate
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
|
|
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
|
|
// says nothing: the runner ended it already.
|
|
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
|
|
started := c.now().UTC()
|
|
ctx, stopWait := context.WithCancel(ctx)
|
|
c.mu.Lock()
|
|
if c.stops == nil {
|
|
c.stops = map[string]context.CancelFunc{}
|
|
}
|
|
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
|
|
c.mu.Unlock()
|
|
stopWait()
|
|
return
|
|
}
|
|
c.stops[id] = stopWait
|
|
c.mu.Unlock()
|
|
defer func() {
|
|
c.mu.Lock()
|
|
delete(c.stops, id)
|
|
c.mu.Unlock()
|
|
stopWait()
|
|
}()
|
|
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
|
|
stop := c.sayProgress(id, run.Asker, phase)
|
|
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
|
stop()
|
|
if ctx.Err() != nil {
|
|
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
|
|
return
|
|
}
|
|
c.sayEnd(id, run, &started, o, again)
|
|
}
|
|
|
|
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
|
|
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
|
|
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
|
|
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
|
|
ended := c.now().UTC()
|
|
if started == nil {
|
|
started = &ended
|
|
}
|
|
step := meshwants.StepEndedStepComposition
|
|
if run.Step == meshwants.StepRunStepValidate {
|
|
step = meshwants.StepEndedStepValidate
|
|
}
|
|
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: step, Module: run.Module,
|
|
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
|
|
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
|
|
msgID := id
|
|
if again != "" {
|
|
msgID = id + "." + again
|
|
}
|
|
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
|
|
c.logf("saying the end of the composition step %s: %v", id, err)
|
|
return
|
|
}
|
|
if c.ended != nil {
|
|
c.ended(id, run, ended)
|
|
}
|
|
}
|
|
|
|
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
|
|
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
|
|
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
|
|
return func(run meshwants.CompositionRun) {
|
|
for _, reader := range run.Readers {
|
|
if !reader.Rerun || c.running(reader.ID) {
|
|
continue
|
|
}
|
|
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
|
|
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
|
|
}
|
|
}
|
|
}
|
|
|
|
// running is whether the step asked under id runs here.
|
|
func (c *compositionSteps) running(id string) bool {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.stops[id] != nil
|
|
}
|
|
|
|
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
|
|
func (c *compositionSteps) leave(ids []string) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
for _, id := range ids {
|
|
if stop := c.stops[id]; stop != nil {
|
|
stop()
|
|
}
|
|
}
|
|
}
|
|
|
|
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
|
|
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
|
|
func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop func()) {
|
|
ticks := c.ticks
|
|
if ticks == nil {
|
|
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
|
|
}
|
|
tick, stopTicks := ticks(progressEvery)
|
|
done := make(chan struct{})
|
|
ended := make(chan struct{})
|
|
go func() {
|
|
defer close(ended)
|
|
defer stopTicks()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-tick:
|
|
at := c.now().UTC()
|
|
p := meshwants.StepProgressEvent{Ask: id, Asker: asker, Phase: phase.get(), At: at, Shared: phase.waitsOn()}
|
|
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
|
|
c.logf("saying the progress of the composition step %s: %v", id, err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return func() { close(done); <-ended }
|
|
}
|
|
|
|
// stepAskID is an ask's id as the asker names it: its event id, else its message id.
|
|
func stepAskID(m *nats.Msg) string {
|
|
if id := m.Header.Get("x-event-id"); id != "" {
|
|
return id
|
|
}
|
|
return m.Header.Get(nats.MsgIdHdr)
|
|
}
|
|
|
|
// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject.
|
|
func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error {
|
|
sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) })
|
|
if err != nil {
|
|
return err
|
|
}
|
|
go func() { <-ctx.Done(); _ = sub.Drain() }()
|
|
return nil
|
|
}
|