Merge pull request 'steprun: only runs that started count toward poison; every end of an ask is written, said and acknowledged (A-489, A-489a, A-489b, A-489c)' (#257) from fix/build-node-step-ends into main
This commit was merged in pull request #257.
This commit is contained in:
@@ -107,16 +107,41 @@ func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) {
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why)
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeFailed, why)
|
||||
_ = msg.Ack() // A-489: ended and written, so it leaves the work queue
|
||||
}
|
||||
|
||||
// sayAskEnded says the end of an ask this holder ends without running it: its step-ended, an ask outside a pipeline as
|
||||
// sayEnded says it, a pipeline step as its own form (design 47 G §602), so the runner that asked hears it.
|
||||
func (p *Puller) sayAskEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) {
|
||||
if ask.Step == nil {
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why, nil, "")
|
||||
} else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil {
|
||||
p.sayEnded(msg, ask, outcome, said, nil, "")
|
||||
} else if end, err := sdkstep.EndOf(ask, p.Node, outcome); err == nil {
|
||||
// A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602).
|
||||
end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask)
|
||||
end.Said, end.At, end.Asker = said, now().UTC(), askerOf(msg, ask)
|
||||
if body, err := sdkstep.EncodeStepEnded(end); err == nil {
|
||||
p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID)
|
||||
}
|
||||
}
|
||||
_ = msg.Term()
|
||||
}
|
||||
|
||||
// PriorRuns is Puller.Prior over a steps state's read (a module's ReadSteps.Get): the Run of the step a key holds. Nil for no
|
||||
// read (a test's), so the bus's delivery count stands.
|
||||
func PriorRuns(get func(key string) (json.RawMessage, error)) func(string) int {
|
||||
if get == nil {
|
||||
return nil
|
||||
}
|
||||
return func(key string) int {
|
||||
raw, err := get(key)
|
||||
if err != nil || raw == nil {
|
||||
return 0
|
||||
}
|
||||
var s sdkstep.Step
|
||||
if json.Unmarshal(raw, &s) != nil {
|
||||
return 0
|
||||
}
|
||||
return s.Run
|
||||
}
|
||||
}
|
||||
|
||||
// giveBack hands an ask back to be judged again a little later, once the seat list says more. Why it cannot be judged is
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) {
|
||||
key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID)
|
||||
waitFor(t, "the step keyed "+key, func() bool { _, ok := st.get(key); return ok })
|
||||
case "refused":
|
||||
waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 })
|
||||
waitFor(t, "the ask ended", func() bool { acked, _, _ := src.Fates(); return len(acked) == 1 }) // A-489: ended, written, acknowledged
|
||||
key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID)
|
||||
s, _ := st.get(key)
|
||||
if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package steprun
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
@@ -56,6 +57,20 @@ type AskSteps struct {
|
||||
SayStep func(end sdkstep.StepEnded) error
|
||||
}
|
||||
|
||||
// A-489b (G-22-lead-answers-2.md): SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to
|
||||
// publish it (design 47 G *Asked on a work queue, ended by an event*, §24). A build module whose steps cannot is refused where
|
||||
// it is built (conformance.StepEnds, which every build module's test runs, and the catalogue check failurekit), never
|
||||
// discovered live as a step ask refused with "says no step's end".
|
||||
func (s AskSteps) SaysItsEnds() error {
|
||||
switch {
|
||||
case s.SayStep == nil:
|
||||
return errors.New("its AskSteps has no SayStep, so a step ask is refused: set it to SayStepOn the seat's language and node")
|
||||
case s.Node == "":
|
||||
return errors.New("its AskSteps names no Node, so the end of a step ask names no node")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunAsk runs ask's job with steps and answers a sdkstep.BuildAnswer or a sdkstep.CheckAnswer.
|
||||
func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, error) {
|
||||
if ask.Build != nil {
|
||||
@@ -150,7 +165,9 @@ func (s AskSteps) fetch(ctx context.Context, fetch *sdkstep.Fetch) (json.RawMess
|
||||
|
||||
// onToolchain is the toolchain this node builds with when the ask names one and this module reports its own, nil when
|
||||
// either is absent. A node on another toolchain than the ask names answers a Declining: it is not on the agreed one and
|
||||
// sits out, for a node that is to take the ask (novox/hq ADR 0584 clause 2).
|
||||
// sits out, for a node that is to take the ask (novox/hq ADR 0584 clause 2). The recipe is compared only when the ask names
|
||||
// one: the runner names a step ask's language and version alone, the recipe and the node's digest being the seat's to report,
|
||||
// not the runner's to know (design 47 G §416, A-104).
|
||||
func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed) (*sdkstep.ToolchainUsed, error) {
|
||||
if want == nil || s.Toolchain == nil {
|
||||
return nil, nil
|
||||
@@ -159,7 +176,7 @@ func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !sdkstep.SameToolchainVersion(tc.Version, want.Version) || tc.Recipe != want.Recipe {
|
||||
if !sdkstep.SameToolchainVersion(tc.Version, want.Version) || (want.Recipe != "" && tc.Recipe != want.Recipe) {
|
||||
return nil, &Declining{Lacks: []string{fmt.Sprintf("the agreed toolchain %s (recipe %s)", sdkstep.LanguageVersion(want.Version), want.Recipe)},
|
||||
Text: fmt.Sprintf("the ask is for %s (recipe %s) and this node builds with %s (recipe %s): it sits out, and the ask is left to a node on the agreed toolchain",
|
||||
sdkstep.LanguageVersion(want.Version), want.Recipe, sdkstep.LanguageVersion(tc.Version), tc.Recipe)}
|
||||
|
||||
@@ -512,3 +512,27 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §416, A-104: the runner names a step ask's toolchain by language and version alone, so a node on that version
|
||||
// takes it whatever its recipe; an ask that names a recipe still declines a node on another.
|
||||
func TestAnAskNamingNoRecipeIsNotDeclinedForItsRecipe(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
want sdkstep.ToolchainUsed
|
||||
decline bool
|
||||
}{
|
||||
{"version, no recipe", sdkstep.ToolchainUsed{Version: "go1.26"}, false},
|
||||
{"the same recipe", sdkstep.ToolchainUsed{Version: "go1.26", Recipe: "3"}, false},
|
||||
{"another recipe", sdkstep.ToolchainUsed{Version: "go1.26", Recipe: "4"}, true},
|
||||
{"another version, no recipe", sdkstep.ToolchainUsed{Version: "go1.27"}, true},
|
||||
} {
|
||||
s := AskSteps{Toolchain: func(context.Context) (sdkstep.ToolchainUsed, error) {
|
||||
return sdkstep.ToolchainUsed{Version: "go1.26.2", Recipe: "3"}, nil
|
||||
}}
|
||||
_, err := s.onToolchain(context.Background(), &c.want)
|
||||
var dec *Declining
|
||||
if errors.As(err, &dec) != c.decline || (err != nil && !c.decline) {
|
||||
t.Errorf("%s: err %v, want declined %v", c.name, err, c.decline)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,10 @@ package conformance
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -277,3 +280,34 @@ func judgePassing(c PassingCase, failed Failed) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// A-489b: StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end
|
||||
// (steprun.AskSteps.SaysItsEnds), or the module would refuse every step ask the runner sends, as mesh-build-node did on its
|
||||
// first live pipeline (design 47 G §24). A catalogue check (failurekit) refuses a build module whose tests do not call it.
|
||||
func StepEnds(t *testing.T, steps steprun.AskSteps) {
|
||||
t.Helper()
|
||||
if err := steps.SaysItsEnds(); err != nil {
|
||||
t.Errorf("the build module's steps cannot say a step's end: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A-489c: RunsCounted is the case every build module runs on its puller (A-489a): the poison rule and a transport fault's run count the
|
||||
// runs the module's steps state says started, so the puller is built with Prior over the module's own read of that state. The
|
||||
// module builds its puller with read as its steps state's read; a step key it holds with a started run must count as that run.
|
||||
func RunsCounted(t *testing.T, puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) {
|
||||
t.Helper()
|
||||
if err := runsCounted(puller); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func runsCounted(puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) error {
|
||||
p := puller(func(string) (json.RawMessage, error) { return json.RawMessage(`{"run":2}`), nil })
|
||||
switch {
|
||||
case p == nil || p.Prior == nil:
|
||||
return errors.New("the build module's puller has no Prior: an ask given back would count toward poison")
|
||||
case p.Prior("any") != 2:
|
||||
return fmt.Errorf("Prior over a step key holding run 2 = %d, want 2: the puller does not read the module's steps state", p.Prior("any"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -181,3 +181,33 @@ func TestTheFailureContract(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A build module whose steps cannot say a step's end fails the case; one that can passes.
|
||||
func TestStepEndsRefusesStepsThatCannotSayTheirEnd(t *testing.T) {
|
||||
say := func(sdkstep.StepEnded) error { return nil }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
s steprun.AskSteps
|
||||
bad bool
|
||||
}{{"no SayStep", steprun.AskSteps{Node: "n"}, true}, {"no node", steprun.AskSteps{SayStep: say}, true}, {"both", steprun.AskSteps{Node: "n", SayStep: say}, false}} {
|
||||
if (c.s.SaysItsEnds() != nil) != c.bad {
|
||||
t.Errorf("%s: SaysItsEnds = %v, want refused %v", c.name, c.s.SaysItsEnds(), c.bad)
|
||||
}
|
||||
}
|
||||
StepEnds(t, steprun.AskSteps{Node: "n", SayStep: say})
|
||||
}
|
||||
|
||||
// A puller built without Prior, or with one that does not read the module's steps state, fails the case.
|
||||
func TestRunsCountedRefusesAPullerThatDoesNotReadTheStepsState(t *testing.T) {
|
||||
wired := func(read func(string) (json.RawMessage, error)) *steprun.Puller {
|
||||
return &steprun.Puller{Prior: steprun.PriorRuns(read)}
|
||||
}
|
||||
if err := runsCounted(wired); err != nil {
|
||||
t.Errorf("a wired puller: %v", err)
|
||||
}
|
||||
for name, p := range map[string]*steprun.Puller{"none": {}, "constant": {Prior: func(string) int { return 0 }}} {
|
||||
if runsCounted(func(func(string) (json.RawMessage, error)) *steprun.Puller { return p }) == nil {
|
||||
t.Errorf("%s: the case passed a puller that does not count the state's runs", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ type Pulled struct {
|
||||
Subject string
|
||||
// Delivered is how many times the bus has handed it over, this time included.
|
||||
Delivered uint64
|
||||
// Sequence is its sequence in the stream (0 for a source that has none), which the journal names for an ask that cannot be read.
|
||||
Sequence uint64
|
||||
// Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its
|
||||
// acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it.
|
||||
Ack, InProgress, Nak, Term func() error
|
||||
@@ -87,7 +89,11 @@ type Puller struct {
|
||||
KeepOutput func(object string, body []byte) error
|
||||
// Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions);
|
||||
// found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163).
|
||||
Callers func(kind string) (callers []string, found bool)
|
||||
Callers func(kind string) (callers []string, found bool)
|
||||
// Prior says how many runs of the ask under a step key started before (the Run of the step the key holds, PriorRuns; 0 for
|
||||
// none), so the poison rule and the transport fault's run count only runs that started, never an ask given back before it
|
||||
// ran. Nil counts the bus's deliveries (a test's).
|
||||
Prior func(key string) int
|
||||
ProgressEvery time.Duration
|
||||
Logf func(format string, args ...any)
|
||||
// Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull
|
||||
@@ -576,7 +582,8 @@ func (p *Puller) decline(msg *Pulled, ask sdkstep.StepAsk, lacks []string) {
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", "))
|
||||
_ = msg.Term()
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "this node lacks "+strings.Join(lacks, ", "))
|
||||
_ = msg.Ack() // A-489a: its end is written
|
||||
p.mu.Lock()
|
||||
if p.declined == nil {
|
||||
p.declined = map[string]bool{}
|
||||
@@ -598,7 +605,7 @@ func (p *Puller) declineRun(msg *Pulled, ask sdkstep.StepAsk, key string, step s
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is declined on %s: %s", ask.ID, p.Node, d.Text)
|
||||
_ = msg.Term()
|
||||
_ = msg.Ack() // A-489a: its end is written (run() says it)
|
||||
p.mu.Lock()
|
||||
if p.declined == nil {
|
||||
p.declined = map[string]bool{}
|
||||
@@ -625,16 +632,18 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) {
|
||||
step := sdkstep.Step{ID: probe.ID, Tool: tool, Node: p.Node, State: sdkstep.StepRefused, Started: at, Ended: at,
|
||||
Reason: fmt.Sprintf("this build module on %s cannot read the ask: %v", p.Node, why), Owner: buildrun.Owner}
|
||||
if werr := writeEnd(p.Put, key, step); werr != nil {
|
||||
p.logf("the unread ask %s cannot be answered on %s: %v", probe.ID, p.Node, werr)
|
||||
p.logf("the unread ask %s cannot be answered on %s, so it is given back: %v", probe.ID, p.Node, werr)
|
||||
_ = msg.Nak() // A-489a: an end that could not be written is given back, never acknowledged
|
||||
return
|
||||
} else {
|
||||
p.logf("an ask refused: %v", why)
|
||||
_ = msg.Term()
|
||||
_ = msg.Ack() // A-489: ended and written, so it leaves the work queue
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
p.logf("an ask dropped: %v", why)
|
||||
_ = msg.Term()
|
||||
p.logf("an ask dropped, no id to answer it by, acknowledged with this line as its only record (stream sequence %d): %v", msg.Sequence, why)
|
||||
_ = msg.Ack() // A-489a
|
||||
}
|
||||
|
||||
func withKind(msg *Pulled, kind string) *Pulled {
|
||||
@@ -644,6 +653,37 @@ func withKind(msg *Pulled, kind string) *Pulled {
|
||||
return msg
|
||||
}
|
||||
|
||||
// priorRuns is how many runs of the ask started before this delivery: its step key's (Puller.Prior), else the bus's delivery
|
||||
// count less this one. The bus's count also counts an ask given back before it ran, which is no run.
|
||||
func (p *Puller) priorRuns(msg *Pulled, ask sdkstep.StepAsk) int {
|
||||
if p.Prior != nil {
|
||||
if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil {
|
||||
return p.Prior(akey)
|
||||
}
|
||||
}
|
||||
return int(msg.Delivered) - 1
|
||||
}
|
||||
|
||||
// endUnrun ends an ask that is run no more (A-489, A-489a): poison when runs of it started and none ended, out of its
|
||||
// deliveries when none did. Its step ends and its step-ended is said, and only then it is acknowledged; an end that could not
|
||||
// be written gives it back, never acknowledged.
|
||||
func (p *Puller) endUnrun(msg *Pulled, ask sdkstep.StepAsk, key string, step sdkstep.Step, prior int) {
|
||||
var why error
|
||||
if prior > 0 {
|
||||
why = fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior)
|
||||
} else {
|
||||
why = fmt.Errorf("the ask is on its last delivery (%d) and no run of it started: it is out of its deliveries and run no more", msg.Delivered)
|
||||
}
|
||||
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
|
||||
if _, ferr := finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf); ferr != nil {
|
||||
p.logf("the end of the ask %s could not be written on %s, so it is given back: %v", ask.ID, p.Node, ferr)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error())
|
||||
_ = msg.Ack()
|
||||
}
|
||||
|
||||
// take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id.
|
||||
func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
kind := msg.Kind
|
||||
@@ -658,6 +698,18 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
p.refuseUnread(msg, err)
|
||||
return
|
||||
}
|
||||
if msg.Delivered >= sdkstep.AskMaxDeliver {
|
||||
// Its last delivery (A-489a): the bus gives it no more, so it is ended here, run or not, never given back.
|
||||
prior := p.priorRuns(msg, ask)
|
||||
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1)
|
||||
if err != nil {
|
||||
p.logf("the ask %s is on its last delivery and cannot start its end on %s: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
p.endUnrun(msg, ask, key, step, prior)
|
||||
return
|
||||
}
|
||||
if why, judged := p.askerRefusal(msg, ask); !judged {
|
||||
p.giveBack(msg, ask)
|
||||
return
|
||||
@@ -673,19 +725,15 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
p.decline(msg, ask, lacks)
|
||||
return
|
||||
}
|
||||
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), int(msg.Delivered))
|
||||
prior := p.priorRuns(msg, ask)
|
||||
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1)
|
||||
if err != nil {
|
||||
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if msg.Delivered > sdkstep.AskRuns {
|
||||
// Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said
|
||||
// (issue 1907): the asker's key fails with why.
|
||||
why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", msg.Delivered-1)
|
||||
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
|
||||
_, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then
|
||||
_ = msg.Term()
|
||||
if prior >= sdkstep.AskRuns {
|
||||
p.endUnrun(msg, ask, key, step, prior)
|
||||
return
|
||||
}
|
||||
module := ""
|
||||
@@ -768,14 +816,14 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
|
||||
var dec *Declining
|
||||
if errors.As(err, &dec) && ctx.Err() == nil {
|
||||
p.declineRun(msg, ask, key, step, dec)
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text, nil, "")
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text)
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs.
|
||||
return
|
||||
}
|
||||
if run := int(msg.Delivered); err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil {
|
||||
if run := step.Run; err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil {
|
||||
// A transport fault (ADR 0664 §1): the step says it is given again and the bus gives it after a delay, to this node or
|
||||
// another; it is not acknowledged, so its asker sees no end. Run AskRuns times it ends failed, said as a transport fault.
|
||||
delay := sdkstep.AgainAfter(run)
|
||||
|
||||
@@ -116,11 +116,11 @@ func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait ti
|
||||
if !ok {
|
||||
return nil, batch.Error()
|
||||
}
|
||||
var delivered uint64 = 1
|
||||
var delivered, sequence uint64 = 1, 0
|
||||
if meta, err := msg.Metadata(); err == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
delivered, sequence = meta.NumDelivered, meta.Sequence.Stream
|
||||
}
|
||||
return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered,
|
||||
return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, Sequence: sequence,
|
||||
Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term,
|
||||
NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil
|
||||
case <-ctx.Done():
|
||||
@@ -146,12 +146,12 @@ func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language str
|
||||
subject, _ := sdkstep.AskSubject(language, kind)
|
||||
// Two workers whose filters do not overlap: main's subject and the asker's, `<main's>.<asker>` (design 47 G §544).
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: sdkstep.AskMaxDeliver}); err != nil {
|
||||
return fmt.Errorf("the worker %s: %w", worker, err)
|
||||
}
|
||||
named, _ := sdkstep.AskNamedWorker(language, kind)
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: sdkstep.AskMaxDeliver}); err != nil {
|
||||
return fmt.Errorf("the worker %s: %w", named, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,7 +147,8 @@ func (p *Puller) refuseDropped(msg *Pulled, ask sdkstep.StepAsk, d *nodebuildgo.
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is refused on %s: it was dropped (%s)", ask.ID, p.Node, d.Why)
|
||||
_ = msg.Term()
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "it was dropped: "+d.Why)
|
||||
_ = msg.Ack() // A-489a: its end is written
|
||||
p.forgetDrop(ask.ID)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// Design 47 G *Asked on a work queue, ended by an event* (§24): the bus counts an ask given back before it ran (a seat list
|
||||
// not read yet) as a delivery, and that is no run. Only runs that started count toward the poison rule (Puller.Prior, from the
|
||||
// step key), and a poisoned pipeline step is ended by its step-ended event, so the runner that asked hears it.
|
||||
func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) {
|
||||
stepAsk := func() sdkstep.StepAsk {
|
||||
return sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(),
|
||||
Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40),
|
||||
Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
delivered uint64
|
||||
prior int
|
||||
ends string // what its end says; "" when it must run
|
||||
}{
|
||||
{"given back three times, never run", sdkstep.AskRuns + 1, 0, ""},
|
||||
{"three runs started and none ended", 3, sdkstep.AskRuns, "poison"},
|
||||
{"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, "poison"},
|
||||
{"its last delivery, given back before and never run", sdkstep.AskMaxDeliver, 0, "out of its deliveries"},
|
||||
{"its last delivery after a run that started", sdkstep.AskMaxDeliver, 1, "poison"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = c.delivered
|
||||
a := stepAsk()
|
||||
raw, err := sdkstep.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src.Publish(sdkstep.AskCheck, raw) // a pipeline step rides the check queue
|
||||
var mu sync.Mutex
|
||||
var ended []sdkstep.StepEnded
|
||||
ran := make(chan struct{}, 1)
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Logf: t.Logf, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Prior: func(string) int { return c.prior },
|
||||
Say: func(event string, body any, id string) error {
|
||||
if event == "step-ended" {
|
||||
var e sdkstep.StepEnded
|
||||
raw, _ := json.Marshal(body)
|
||||
if err := json.Unmarshal(raw, &e); err == nil {
|
||||
mu.Lock()
|
||||
ended = append(ended, e)
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
if c.ends != "" {
|
||||
waitFor(t, "the poisoned ask's end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) == 1 })
|
||||
mu.Lock()
|
||||
e := ended[0]
|
||||
mu.Unlock()
|
||||
if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, c.ends) || e.Pipeline != "main.12.m" {
|
||||
t.Fatalf("the end said %+v, want could not run, poison, naming its pipeline", e)
|
||||
}
|
||||
if acked, termed, naked := src.Fates(); len(acked) != 1 || len(termed)+len(naked) != 0 {
|
||||
t.Errorf("acked %v termed %v naked %v, want it acknowledged once its end is written (A-489a)", acked, termed, naked)
|
||||
}
|
||||
select {
|
||||
case <-ran:
|
||||
t.Fatal("an ended ask ran")
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ran:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("an ask only given back was ended as poison, and never ran")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A-489a: an end that could not be written is given back, never acknowledged: the ask's start is written, its end is not
|
||||
// (poison), and an unreadable ask with an id whose refusal cannot be written.
|
||||
func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) {
|
||||
was := EndAttempts
|
||||
EndAttempts = 1
|
||||
defer func() { EndAttempts = was }()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
data func() []byte
|
||||
prior int
|
||||
}{
|
||||
{"poison", func() []byte { raw, _ := sdkstep.EncodeStepAsk(testAsk(sdkstep.AskDelivery)); return raw }, sdkstep.AskRuns},
|
||||
{"unread with an id", func() []byte { return []byte(`{"id":"mwn1abcdef0123","kind":"check","v":99}`) }, 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
src := NewFakeAsks()
|
||||
src.Publish(sdkstep.AskCheck, c.data())
|
||||
if c.prior > 0 { // a delivery of the delivery queue
|
||||
src = NewFakeAsks()
|
||||
src.Publish(sdkstep.AskDelivery, c.data())
|
||||
}
|
||||
var mu sync.Mutex
|
||||
puts := 0
|
||||
put := func(string, any) (uint64, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
puts++
|
||||
if puts > 1 || c.prior == 0 { // the start is written, the end is not
|
||||
return 0, errors.New("the state is full")
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
|
||||
Prior: func(string) int { return c.prior }, Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask given back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 })
|
||||
if acked, termed, _ := src.Fates(); len(acked)+len(termed) != 0 {
|
||||
t.Errorf("acked %v termed %v, want none: no end was written", acked, termed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A-489a: a decline at the take (a need the node lacks), a refused run (Declining) and a dropped ask each say their pipeline step's end as could not
|
||||
// run, naming its pipeline, and are acknowledged once it is written.
|
||||
func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) {
|
||||
for _, c := range []string{"declined", "declined in its run", "dropped"} {
|
||||
t.Run(c, func(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
a := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(),
|
||||
Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40),
|
||||
Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}}
|
||||
raw, eerr := sdkstep.EncodeStepAsk(a)
|
||||
if eerr != nil {
|
||||
t.Fatal(eerr)
|
||||
}
|
||||
src.Publish(sdkstep.AskCheck, raw)
|
||||
var mu sync.Mutex
|
||||
var ended []sdkstep.StepEnded
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
|
||||
Say: func(event string, body any, id string) error {
|
||||
if event == "step-ended" {
|
||||
var e sdkstep.StepEnded
|
||||
b, _ := json.Marshal(body)
|
||||
if json.Unmarshal(b, &e) == nil {
|
||||
mu.Lock()
|
||||
ended = append(ended, e)
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Run: func(context.Context, sdkstep.StepAsk) (any, error) {
|
||||
return nil, &Declining{Text: "this node builds with another toolchain", Lacks: []string{"the toolchain"}}
|
||||
}}
|
||||
if c == "dropped" {
|
||||
drop, _ := json.Marshal(nodebuildgo.Drop{V: int64(sdkstep.Version), By: "mesh-delivery-builder", Why: "superseded", At: ptr(time.Now())})
|
||||
p.Load = func(key string) ([]byte, error) {
|
||||
if key == sdkstep.DropKey(a.ID) {
|
||||
return drop, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
if c == "declined" {
|
||||
// The decoder refuses needs on a pipeline step ask, so no ask of the queue reaches the decline; the line is held
|
||||
// by calling it with the ask built whole.
|
||||
acked := 0
|
||||
p.decline(&Pulled{Ack: func() error { acked++; return nil }, Term: func() error { return nil }, Nak: func() error { return nil }}, a, []string{"node-build-node"})
|
||||
if acked != 1 {
|
||||
t.Errorf("acked %d times, want once", acked)
|
||||
}
|
||||
src.Acked = append(src.Acked, "declined")
|
||||
} else {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
}
|
||||
waitFor(t, "the end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) >= 1 })
|
||||
waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 })
|
||||
mu.Lock()
|
||||
e := ended[0]
|
||||
mu.Unlock()
|
||||
if e.Outcome != sdkstep.OutcomeCouldNotRun || e.Pipeline != "main.12.m" {
|
||||
t.Errorf("the end said %+v, want could not run naming its pipeline", e)
|
||||
}
|
||||
if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 {
|
||||
t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the
|
||||
// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after 4 minutes), not 3.
|
||||
func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = 3
|
||||
a := testAsk(sdkstep.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
|
||||
Prior: func(string) int { return 1 },
|
||||
Run: func(context.Context, sdkstep.StepAsk) (any, error) {
|
||||
return nil, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask given again", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) == 1 })
|
||||
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
|
||||
s, _ := st.get(key)
|
||||
src.mu.Lock()
|
||||
delays := append([]time.Duration(nil), src.Delays...)
|
||||
src.mu.Unlock()
|
||||
if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != sdkstep.AgainAfter(2) {
|
||||
t.Errorf("step %+v, delays %v, want again in run 2 after %s", s, delays, sdkstep.AgainAfter(2))
|
||||
}
|
||||
}
|
||||
|
||||
// Puller.Prior over a steps state's read: the Run of the step the key holds, 0 for an absent key, a failed read or what is no
|
||||
// step, and nil for no read.
|
||||
func TestPriorRunsReadsTheRunOfTheStepAKeyHolds(t *testing.T) {
|
||||
read := func(key string) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case "running":
|
||||
return json.RawMessage(`{"id":"x","run":2}`), nil
|
||||
case "junk":
|
||||
return json.RawMessage(`[`), nil
|
||||
case "broken":
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
prior := PriorRuns(read)
|
||||
for key, want := range map[string]int{"running": 2, "absent": 0, "junk": 0, "broken": 0} {
|
||||
if got := prior(key); got != want {
|
||||
t.Errorf("PriorRuns(%s) = %d, want %d", key, got, want)
|
||||
}
|
||||
}
|
||||
if PriorRuns(nil) != nil {
|
||||
t.Error("PriorRuns(nil) is not nil")
|
||||
}
|
||||
}
|
||||
@@ -244,7 +244,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "three termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 3 })
|
||||
waitFor(t, "three acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 3 })
|
||||
if ran {
|
||||
t.Fatal("a wrong or a declined ask ran")
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) {
|
||||
return nil, &Declining{Text: "the ask is for 26.13.0 and this node builds with 26.12.1", Lacks: []string{"the agreed toolchain 26.13.0 (recipe 3)"}}
|
||||
}}
|
||||
go s2.Pull(ctx)
|
||||
waitFor(t, "the ask termed on the node that sits out", func() bool { _, tm, _ := sit.Fates(); return len(tm) == 1 })
|
||||
waitFor(t, "the ask acknowledged on the node that sits out", func() bool { ak, _, _ := sit.Fates(); return len(ak) == 1 })
|
||||
sk, _ := sdkstep.AskStepKey("", "ace", toolchain.ID)
|
||||
ss, _ := sitSteps.get(sk)
|
||||
if d, ok := sdkstep.DeclinedOf(ss); !ok || len(d.Lacks) != 1 || !strings.Contains(ss.Reason, "26.12.1") {
|
||||
@@ -538,14 +538,14 @@ func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 1 })
|
||||
waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 })
|
||||
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
|
||||
s, ok := st.get(key)
|
||||
if ran || !ok || s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") {
|
||||
t.Fatalf("ran %v, step %+v", ran, s)
|
||||
}
|
||||
if acked, _, naked := src.Fates(); len(acked)+len(naked) != 0 {
|
||||
t.Errorf("acked %v naked %v, want it termed alone", acked, naked)
|
||||
if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 { // A-489: its end written, it leaves the queue by ack
|
||||
t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked)
|
||||
}
|
||||
// ADR 0664 §1 (issue 1748): a run that ends on a transport fault below AskRuns is written again, with its run and when the
|
||||
// bus gives it, and handed back with that delay, not acknowledged; the AskRuns-th ends failed, passing; a fault of the
|
||||
@@ -620,7 +620,7 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "both termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 })
|
||||
waitFor(t, "both acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 2 })
|
||||
key, _ := sdkstep.AskStepKey("", "ace", good.ID)
|
||||
s, ok := st.get(key)
|
||||
if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") {
|
||||
|
||||
Reference in New Issue
Block a user