Compare commits

...
Author SHA1 Message Date
jschoubben 7617dda5f6 steprun: an ask naming no recipe is not declined for its recipe
The runner names a step ask's toolchain by language and version; every node sat out every TypeScript step.

Builds: design 47 G §416 (A-104)

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 16:27:26 +01:00
jschoubben 13db05ce60 steprun: the decline of a pipeline step is held by a test; RunsCounted cites A-489c
Builds: design 47 G §24; lead answers A-489a, A-489c

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 15:45:48 +01:00
jschoubben 334bc54767 steprun: an unread ask whose refusal cannot be written is given back; the unwritten-end and said-end cases assert what they claim; kit case RunsCounted
Builds: design 47 G §24; lead answers A-489a, A-489b

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 15:42:16 +01:00
jschoubben 92196dde3a steprun: an ask on its last delivery is ended, every end is acknowledged, an unwritten end is given back (A-489a)
Builds: design 47 G §24, §25; lead answers A-489, A-489a

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 15:38:16 +01:00
jschoubben 16feb56fd4 steprun: PriorRuns reads a module's ReadSteps shape
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 15:24:37 +01:00
jschoubben 78adc23d70 steprun: only runs that started count toward poison; an ended ask is acknowledged and its poison end said (A-489)
The bus counts an ask given back before it ran as a delivery, so four asks were ended 'poison' having never run. Puller.Prior
(PriorRuns) counts the runs the step key says started. A poisoned pipeline step now says its step-ended, so the runner stops
waiting. refuseUnread, refuseAsker and the poison path acknowledge the ask once its end is written, not terminate it.

Builds: design 47 G §24, §25; lead answer A-489 (G-22-lead-answers-2.md)

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 15:23:13 +01:00
jschoubben 79bca88c8a steprun: a build module whose steps cannot say a step's end fails its own test (conformance.StepEnds)
A step ask refused live with 'this build module says no step's end' (mesh-build-node set no SayStep). AskSteps.SaysItsEnds
and the kit case conformance.StepEnds refuse that where the module is built.

Builds: design 47 G §24 (Asked on a work queue, ended by an event)

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 15:23:13 +01:00
11 changed files with 473 additions and 36 deletions
+29 -4
View File
@@ -107,16 +107,41 @@ func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) {
return return
} }
p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why) p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why)
p.sayAskEnded(msg, ask, sdkstep.OutcomeFailed, why)
_ = msg.Ack() // A-489: ended and written, so it leaves the work queue
}
// sayAskEnded says the end of an ask this holder ends without running it: its step-ended, an ask outside a pipeline as
// sayEnded says it, a pipeline step as its own form (design 47 G §602), so the runner that asked hears it.
func (p *Puller) sayAskEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) {
if ask.Step == nil { if ask.Step == nil {
p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why, nil, "") p.sayEnded(msg, ask, outcome, said, nil, "")
} else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil { } else if end, err := sdkstep.EndOf(ask, p.Node, outcome); err == nil {
// A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602). // A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602).
end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask) end.Said, end.At, end.Asker = said, now().UTC(), askerOf(msg, ask)
if body, err := sdkstep.EncodeStepEnded(end); err == nil { if body, err := sdkstep.EncodeStepEnded(end); err == nil {
p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID)
} }
} }
_ = msg.Term() }
// PriorRuns is Puller.Prior over a steps state's read (a module's ReadSteps.Get): the Run of the step a key holds. Nil for no
// read (a test's), so the bus's delivery count stands.
func PriorRuns(get func(key string) (json.RawMessage, error)) func(string) int {
if get == nil {
return nil
}
return func(key string) int {
raw, err := get(key)
if err != nil || raw == nil {
return 0
}
var s sdkstep.Step
if json.Unmarshal(raw, &s) != nil {
return 0
}
return s.Run
}
} }
// giveBack hands an ask back to be judged again a little later, once the seat list says more. Why it cannot be judged is // giveBack hands an ask back to be judged again a little later, once the seat list says more. Why it cannot be judged is
@@ -65,7 +65,7 @@ func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) {
key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID) key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID)
waitFor(t, "the step keyed "+key, func() bool { _, ok := st.get(key); return ok }) waitFor(t, "the step keyed "+key, func() bool { _, ok := st.get(key); return ok })
case "refused": case "refused":
waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 }) waitFor(t, "the ask ended", func() bool { acked, _, _ := src.Fates(); return len(acked) == 1 }) // A-489: ended, written, acknowledged
key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID) key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID)
s, _ := st.get(key) s, _ := st.get(key)
if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) { if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) {
+19 -2
View File
@@ -3,6 +3,7 @@ package steprun
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"maps" "maps"
"path" "path"
@@ -56,6 +57,20 @@ type AskSteps struct {
SayStep func(end sdkstep.StepEnded) error SayStep func(end sdkstep.StepEnded) error
} }
// A-489b (G-22-lead-answers-2.md): SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to
// publish it (design 47 G *Asked on a work queue, ended by an event*, §24). A build module whose steps cannot is refused where
// it is built (conformance.StepEnds, which every build module's test runs, and the catalogue check failurekit), never
// discovered live as a step ask refused with "says no step's end".
func (s AskSteps) SaysItsEnds() error {
switch {
case s.SayStep == nil:
return errors.New("its AskSteps has no SayStep, so a step ask is refused: set it to SayStepOn the seat's language and node")
case s.Node == "":
return errors.New("its AskSteps names no Node, so the end of a step ask names no node")
}
return nil
}
// RunAsk runs ask's job with steps and answers a sdkstep.BuildAnswer or a sdkstep.CheckAnswer. // RunAsk runs ask's job with steps and answers a sdkstep.BuildAnswer or a sdkstep.CheckAnswer.
func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, error) { func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, error) {
if ask.Build != nil { if ask.Build != nil {
@@ -150,7 +165,9 @@ func (s AskSteps) fetch(ctx context.Context, fetch *sdkstep.Fetch) (json.RawMess
// onToolchain is the toolchain this node builds with when the ask names one and this module reports its own, nil when // onToolchain is the toolchain this node builds with when the ask names one and this module reports its own, nil when
// either is absent. A node on another toolchain than the ask names answers a Declining: it is not on the agreed one and // either is absent. A node on another toolchain than the ask names answers a Declining: it is not on the agreed one and
// sits out, for a node that is to take the ask (novox/hq ADR 0584 clause 2). // sits out, for a node that is to take the ask (novox/hq ADR 0584 clause 2). The recipe is compared only when the ask names
// one: the runner names a step ask's language and version alone, the recipe and the node's digest being the seat's to report,
// not the runner's to know (design 47 G §416, A-104).
func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed) (*sdkstep.ToolchainUsed, error) { func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed) (*sdkstep.ToolchainUsed, error) {
if want == nil || s.Toolchain == nil { if want == nil || s.Toolchain == nil {
return nil, nil return nil, nil
@@ -159,7 +176,7 @@ func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !sdkstep.SameToolchainVersion(tc.Version, want.Version) || tc.Recipe != want.Recipe { if !sdkstep.SameToolchainVersion(tc.Version, want.Version) || (want.Recipe != "" && tc.Recipe != want.Recipe) {
return nil, &Declining{Lacks: []string{fmt.Sprintf("the agreed toolchain %s (recipe %s)", sdkstep.LanguageVersion(want.Version), want.Recipe)}, return nil, &Declining{Lacks: []string{fmt.Sprintf("the agreed toolchain %s (recipe %s)", sdkstep.LanguageVersion(want.Version), want.Recipe)},
Text: fmt.Sprintf("the ask is for %s (recipe %s) and this node builds with %s (recipe %s): it sits out, and the ask is left to a node on the agreed toolchain", Text: fmt.Sprintf("the ask is for %s (recipe %s) and this node builds with %s (recipe %s): it sits out, and the ask is left to a node on the agreed toolchain",
sdkstep.LanguageVersion(want.Version), want.Recipe, sdkstep.LanguageVersion(tc.Version), tc.Recipe)} sdkstep.LanguageVersion(want.Version), want.Recipe, sdkstep.LanguageVersion(tc.Version), tc.Recipe)}
@@ -512,3 +512,27 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
} }
} }
} }
// Design 47 G §416, A-104: the runner names a step ask's toolchain by language and version alone, so a node on that version
// takes it whatever its recipe; an ask that names a recipe still declines a node on another.
func TestAnAskNamingNoRecipeIsNotDeclinedForItsRecipe(t *testing.T) {
for _, c := range []struct {
name string
want sdkstep.ToolchainUsed
decline bool
}{
{"version, no recipe", sdkstep.ToolchainUsed{Version: "go1.26"}, false},
{"the same recipe", sdkstep.ToolchainUsed{Version: "go1.26", Recipe: "3"}, false},
{"another recipe", sdkstep.ToolchainUsed{Version: "go1.26", Recipe: "4"}, true},
{"another version, no recipe", sdkstep.ToolchainUsed{Version: "go1.27"}, true},
} {
s := AskSteps{Toolchain: func(context.Context) (sdkstep.ToolchainUsed, error) {
return sdkstep.ToolchainUsed{Version: "go1.26.2", Recipe: "3"}, nil
}}
_, err := s.onToolchain(context.Background(), &c.want)
var dec *Declining
if errors.As(err, &dec) != c.decline || (err != nil && !c.decline) {
t.Errorf("%s: err %v, want declined %v", c.name, err, c.decline)
}
}
}
@@ -11,7 +11,10 @@ package conformance
import ( import (
"embed" "embed"
"encoding/json"
"errors"
"fmt" "fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"slices" "slices"
"strings" "strings"
"testing" "testing"
@@ -277,3 +280,34 @@ func judgePassing(c PassingCase, failed Failed) string {
} }
return "" return ""
} }
// A-489b: StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end
// (steprun.AskSteps.SaysItsEnds), or the module would refuse every step ask the runner sends, as mesh-build-node did on its
// first live pipeline (design 47 G §24). A catalogue check (failurekit) refuses a build module whose tests do not call it.
func StepEnds(t *testing.T, steps steprun.AskSteps) {
t.Helper()
if err := steps.SaysItsEnds(); err != nil {
t.Errorf("the build module's steps cannot say a step's end: %v", err)
}
}
// A-489c: RunsCounted is the case every build module runs on its puller (A-489a): the poison rule and a transport fault's run count the
// runs the module's steps state says started, so the puller is built with Prior over the module's own read of that state. The
// module builds its puller with read as its steps state's read; a step key it holds with a started run must count as that run.
func RunsCounted(t *testing.T, puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) {
t.Helper()
if err := runsCounted(puller); err != nil {
t.Error(err)
}
}
func runsCounted(puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) error {
p := puller(func(string) (json.RawMessage, error) { return json.RawMessage(`{"run":2}`), nil })
switch {
case p == nil || p.Prior == nil:
return errors.New("the build module's puller has no Prior: an ask given back would count toward poison")
case p.Prior("any") != 2:
return fmt.Errorf("Prior over a step key holding run 2 = %d, want 2: the puller does not read the module's steps state", p.Prior("any"))
}
return nil
}
@@ -181,3 +181,33 @@ func TestTheFailureContract(t *testing.T) {
} }
}) })
} }
// A build module whose steps cannot say a step's end fails the case; one that can passes.
func TestStepEndsRefusesStepsThatCannotSayTheirEnd(t *testing.T) {
say := func(sdkstep.StepEnded) error { return nil }
for _, c := range []struct {
name string
s steprun.AskSteps
bad bool
}{{"no SayStep", steprun.AskSteps{Node: "n"}, true}, {"no node", steprun.AskSteps{SayStep: say}, true}, {"both", steprun.AskSteps{Node: "n", SayStep: say}, false}} {
if (c.s.SaysItsEnds() != nil) != c.bad {
t.Errorf("%s: SaysItsEnds = %v, want refused %v", c.name, c.s.SaysItsEnds(), c.bad)
}
}
StepEnds(t, steprun.AskSteps{Node: "n", SayStep: say})
}
// A puller built without Prior, or with one that does not read the module's steps state, fails the case.
func TestRunsCountedRefusesAPullerThatDoesNotReadTheStepsState(t *testing.T) {
wired := func(read func(string) (json.RawMessage, error)) *steprun.Puller {
return &steprun.Puller{Prior: steprun.PriorRuns(read)}
}
if err := runsCounted(wired); err != nil {
t.Errorf("a wired puller: %v", err)
}
for name, p := range map[string]*steprun.Puller{"none": {}, "constant": {Prior: func(string) int { return 0 }}} {
if runsCounted(func(func(string) (json.RawMessage, error)) *steprun.Puller { return p }) == nil {
t.Errorf("%s: the case passed a puller that does not count the state's runs", name)
}
}
}
+64 -16
View File
@@ -32,6 +32,8 @@ type Pulled struct {
Subject string Subject string
// Delivered is how many times the bus has handed it over, this time included. // Delivered is how many times the bus has handed it over, this time included.
Delivered uint64 Delivered uint64
// Sequence is its sequence in the stream (0 for a source that has none), which the journal names for an ask that cannot be read.
Sequence uint64
// Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its // Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its
// acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it. // acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it.
Ack, InProgress, Nak, Term func() error Ack, InProgress, Nak, Term func() error
@@ -88,6 +90,10 @@ type Puller struct {
// Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions); // Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions);
// found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163). // found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163).
Callers func(kind string) (callers []string, found bool) Callers func(kind string) (callers []string, found bool)
// Prior says how many runs of the ask under a step key started before (the Run of the step the key holds, PriorRuns; 0 for
// none), so the poison rule and the transport fault's run count only runs that started, never an ask given back before it
// ran. Nil counts the bus's deliveries (a test's).
Prior func(key string) int
ProgressEvery time.Duration ProgressEvery time.Duration
Logf func(format string, args ...any) Logf func(format string, args ...any)
// Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull // Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull
@@ -576,7 +582,8 @@ func (p *Puller) decline(msg *Pulled, ask sdkstep.StepAsk, lacks []string) {
return return
} }
p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", ")) p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", "))
_ = msg.Term() p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "this node lacks "+strings.Join(lacks, ", "))
_ = msg.Ack() // A-489a: its end is written
p.mu.Lock() p.mu.Lock()
if p.declined == nil { if p.declined == nil {
p.declined = map[string]bool{} p.declined = map[string]bool{}
@@ -598,7 +605,7 @@ func (p *Puller) declineRun(msg *Pulled, ask sdkstep.StepAsk, key string, step s
return return
} }
p.logf("the ask %s is declined on %s: %s", ask.ID, p.Node, d.Text) p.logf("the ask %s is declined on %s: %s", ask.ID, p.Node, d.Text)
_ = msg.Term() _ = msg.Ack() // A-489a: its end is written (run() says it)
p.mu.Lock() p.mu.Lock()
if p.declined == nil { if p.declined == nil {
p.declined = map[string]bool{} p.declined = map[string]bool{}
@@ -625,16 +632,18 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) {
step := sdkstep.Step{ID: probe.ID, Tool: tool, Node: p.Node, State: sdkstep.StepRefused, Started: at, Ended: at, step := sdkstep.Step{ID: probe.ID, Tool: tool, Node: p.Node, State: sdkstep.StepRefused, Started: at, Ended: at,
Reason: fmt.Sprintf("this build module on %s cannot read the ask: %v", p.Node, why), Owner: buildrun.Owner} Reason: fmt.Sprintf("this build module on %s cannot read the ask: %v", p.Node, why), Owner: buildrun.Owner}
if werr := writeEnd(p.Put, key, step); werr != nil { if werr := writeEnd(p.Put, key, step); werr != nil {
p.logf("the unread ask %s cannot be answered on %s: %v", probe.ID, p.Node, werr) p.logf("the unread ask %s cannot be answered on %s, so it is given back: %v", probe.ID, p.Node, werr)
_ = msg.Nak() // A-489a: an end that could not be written is given back, never acknowledged
return
} else { } else {
p.logf("an ask refused: %v", why) p.logf("an ask refused: %v", why)
_ = msg.Term() _ = msg.Ack() // A-489: ended and written, so it leaves the work queue
return return
} }
} }
} }
p.logf("an ask dropped: %v", why) p.logf("an ask dropped, no id to answer it by, acknowledged with this line as its only record (stream sequence %d): %v", msg.Sequence, why)
_ = msg.Term() _ = msg.Ack() // A-489a
} }
func withKind(msg *Pulled, kind string) *Pulled { func withKind(msg *Pulled, kind string) *Pulled {
@@ -644,6 +653,37 @@ func withKind(msg *Pulled, kind string) *Pulled {
return msg return msg
} }
// priorRuns is how many runs of the ask started before this delivery: its step key's (Puller.Prior), else the bus's delivery
// count less this one. The bus's count also counts an ask given back before it ran, which is no run.
func (p *Puller) priorRuns(msg *Pulled, ask sdkstep.StepAsk) int {
if p.Prior != nil {
if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil {
return p.Prior(akey)
}
}
return int(msg.Delivered) - 1
}
// endUnrun ends an ask that is run no more (A-489, A-489a): poison when runs of it started and none ended, out of its
// deliveries when none did. Its step ends and its step-ended is said, and only then it is acknowledged; an end that could not
// be written gives it back, never acknowledged.
func (p *Puller) endUnrun(msg *Pulled, ask sdkstep.StepAsk, key string, step sdkstep.Step, prior int) {
var why error
if prior > 0 {
why = fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior)
} else {
why = fmt.Errorf("the ask is on its last delivery (%d) and no run of it started: it is out of its deliveries and run no more", msg.Delivered)
}
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
if _, ferr := finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf); ferr != nil {
p.logf("the end of the ask %s could not be written on %s, so it is given back: %v", ask.ID, p.Node, ferr)
_ = msg.Nak()
return
}
p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error())
_ = msg.Ack()
}
// take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id. // take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id.
func (p *Puller) take(ctx context.Context, msg *Pulled) { func (p *Puller) take(ctx context.Context, msg *Pulled) {
kind := msg.Kind kind := msg.Kind
@@ -658,6 +698,18 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
p.refuseUnread(msg, err) p.refuseUnread(msg, err)
return return
} }
if msg.Delivered >= sdkstep.AskMaxDeliver {
// Its last delivery (A-489a): the bus gives it no more, so it is ended here, run or not, never given back.
prior := p.priorRuns(msg, ask)
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1)
if err != nil {
p.logf("the ask %s is on its last delivery and cannot start its end on %s: %v", ask.ID, p.Node, err)
_ = msg.Nak()
return
}
p.endUnrun(msg, ask, key, step, prior)
return
}
if why, judged := p.askerRefusal(msg, ask); !judged { if why, judged := p.askerRefusal(msg, ask); !judged {
p.giveBack(msg, ask) p.giveBack(msg, ask)
return return
@@ -673,19 +725,15 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
p.decline(msg, ask, lacks) p.decline(msg, ask, lacks)
return return
} }
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), int(msg.Delivered)) prior := p.priorRuns(msg, ask)
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1)
if err != nil { if err != nil {
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err) p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
_ = msg.Nak() _ = msg.Nak()
return return
} }
if msg.Delivered > sdkstep.AskRuns { if prior >= sdkstep.AskRuns {
// Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said p.endUnrun(msg, ask, key, step, prior)
// (issue 1907): the asker's key fails with why.
why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", msg.Delivered-1)
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
_, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then
_ = msg.Term()
return return
} }
module := "" module := ""
@@ -768,14 +816,14 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
var dec *Declining var dec *Declining
if errors.As(err, &dec) && ctx.Err() == nil { if errors.As(err, &dec) && ctx.Err() == nil {
p.declineRun(msg, ask, key, step, dec) p.declineRun(msg, ask, key, step, dec)
p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text, nil, "") p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text)
return return
} }
if ctx.Err() != nil { if ctx.Err() != nil {
// The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs. // The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs.
return return
} }
if run := int(msg.Delivered); err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil { if run := step.Run; err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil {
// A transport fault (ADR 0664 §1): the step says it is given again and the bus gives it after a delay, to this node or // A transport fault (ADR 0664 §1): the step says it is given again and the bus gives it after a delay, to this node or
// another; it is not acknowledged, so its asker sees no end. Run AskRuns times it ends failed, said as a transport fault. // another; it is not acknowledged, so its asker sees no end. Run AskRuns times it ends failed, said as a transport fault.
delay := sdkstep.AgainAfter(run) delay := sdkstep.AgainAfter(run)
@@ -116,11 +116,11 @@ func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait ti
if !ok { if !ok {
return nil, batch.Error() return nil, batch.Error()
} }
var delivered uint64 = 1 var delivered, sequence uint64 = 1, 0
if meta, err := msg.Metadata(); err == nil && meta != nil { if meta, err := msg.Metadata(); err == nil && meta != nil {
delivered = meta.NumDelivered delivered, sequence = meta.NumDelivered, meta.Sequence.Stream
} }
return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, Sequence: sequence,
Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term, Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term,
NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil
case <-ctx.Done(): case <-ctx.Done():
@@ -146,12 +146,12 @@ func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language str
subject, _ := sdkstep.AskSubject(language, kind) subject, _ := sdkstep.AskSubject(language, kind)
// Two workers whose filters do not overlap: main's subject and the asker's, `<main's>.<asker>` (design 47 G §544). // Two workers whose filters do not overlap: main's subject and the asker's, `<main's>.<asker>` (design 47 G §544).
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject}, if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: sdkstep.AskMaxDeliver}); err != nil {
return fmt.Errorf("the worker %s: %w", worker, err) return fmt.Errorf("the worker %s: %w", worker, err)
} }
named, _ := sdkstep.AskNamedWorker(language, kind) named, _ := sdkstep.AskNamedWorker(language, kind)
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"}, if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: sdkstep.AskMaxDeliver}); err != nil {
return fmt.Errorf("the worker %s: %w", named, err) return fmt.Errorf("the worker %s: %w", named, err)
} }
} }
@@ -147,7 +147,8 @@ func (p *Puller) refuseDropped(msg *Pulled, ask sdkstep.StepAsk, d *nodebuildgo.
return return
} }
p.logf("the ask %s is refused on %s: it was dropped (%s)", ask.ID, p.Node, d.Why) p.logf("the ask %s is refused on %s: it was dropped (%s)", ask.ID, p.Node, d.Why)
_ = msg.Term() p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "it was dropped: "+d.Why)
_ = msg.Ack() // A-489a: its end is written
p.forgetDrop(ask.ID) p.forgetDrop(ask.ID)
} }
@@ -0,0 +1,258 @@
package steprun
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// Design 47 G *Asked on a work queue, ended by an event* (§24): the bus counts an ask given back before it ran (a seat list
// not read yet) as a delivery, and that is no run. Only runs that started count toward the poison rule (Puller.Prior, from the
// step key), and a poisoned pipeline step is ended by its step-ended event, so the runner that asked hears it.
func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) {
stepAsk := func() sdkstep.StepAsk {
return sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(),
Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40),
Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}}
}
for _, c := range []struct {
name string
delivered uint64
prior int
ends string // what its end says; "" when it must run
}{
{"given back three times, never run", sdkstep.AskRuns + 1, 0, ""},
{"three runs started and none ended", 3, sdkstep.AskRuns, "poison"},
{"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, "poison"},
{"its last delivery, given back before and never run", sdkstep.AskMaxDeliver, 0, "out of its deliveries"},
{"its last delivery after a run that started", sdkstep.AskMaxDeliver, 1, "poison"},
} {
t.Run(c.name, func(t *testing.T) {
src, st := NewFakeAsks(), newSteps()
src.Delivered = c.delivered
a := stepAsk()
raw, err := sdkstep.EncodeStepAsk(a)
if err != nil {
t.Fatal(err)
}
src.Publish(sdkstep.AskCheck, raw) // a pipeline step rides the check queue
var mu sync.Mutex
var ended []sdkstep.StepEnded
ran := make(chan struct{}, 1)
p := &Puller{Source: src, Language: sdkstep.AskGo, Logf: t.Logf, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Prior: func(string) int { return c.prior },
Say: func(event string, body any, id string) error {
if event == "step-ended" {
var e sdkstep.StepEnded
raw, _ := json.Marshal(body)
if err := json.Unmarshal(raw, &e); err == nil {
mu.Lock()
ended = append(ended, e)
mu.Unlock()
}
}
return nil
},
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go p.Pull(ctx)
if c.ends != "" {
waitFor(t, "the poisoned ask's end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) == 1 })
mu.Lock()
e := ended[0]
mu.Unlock()
if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, c.ends) || e.Pipeline != "main.12.m" {
t.Fatalf("the end said %+v, want could not run, poison, naming its pipeline", e)
}
if acked, termed, naked := src.Fates(); len(acked) != 1 || len(termed)+len(naked) != 0 {
t.Errorf("acked %v termed %v naked %v, want it acknowledged once its end is written (A-489a)", acked, termed, naked)
}
select {
case <-ran:
t.Fatal("an ended ask ran")
default:
}
return
}
select {
case <-ran:
case <-time.After(5 * time.Second):
t.Fatal("an ask only given back was ended as poison, and never ran")
}
})
}
}
// A-489a: an end that could not be written is given back, never acknowledged: the ask's start is written, its end is not
// (poison), and an unreadable ask with an id whose refusal cannot be written.
func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) {
was := EndAttempts
EndAttempts = 1
defer func() { EndAttempts = was }()
for _, c := range []struct {
name string
data func() []byte
prior int
}{
{"poison", func() []byte { raw, _ := sdkstep.EncodeStepAsk(testAsk(sdkstep.AskDelivery)); return raw }, sdkstep.AskRuns},
{"unread with an id", func() []byte { return []byte(`{"id":"mwn1abcdef0123","kind":"check","v":99}`) }, 0},
} {
t.Run(c.name, func(t *testing.T) {
src := NewFakeAsks()
src.Publish(sdkstep.AskCheck, c.data())
if c.prior > 0 { // a delivery of the delivery queue
src = NewFakeAsks()
src.Publish(sdkstep.AskDelivery, c.data())
}
var mu sync.Mutex
puts := 0
put := func(string, any) (uint64, error) {
mu.Lock()
defer mu.Unlock()
puts++
if puts > 1 || c.prior == 0 { // the start is written, the end is not
return 0, errors.New("the state is full")
}
return 1, nil
}
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
Prior: func(string) int { return c.prior }, Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, nil }}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go p.Pull(ctx)
waitFor(t, "the ask given back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 })
if acked, termed, _ := src.Fates(); len(acked)+len(termed) != 0 {
t.Errorf("acked %v termed %v, want none: no end was written", acked, termed)
}
})
}
}
// A-489a: a decline at the take (a need the node lacks), a refused run (Declining) and a dropped ask each say their pipeline step's end as could not
// run, naming its pipeline, and are acknowledged once it is written.
func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) {
for _, c := range []string{"declined", "declined in its run", "dropped"} {
t.Run(c, func(t *testing.T) {
src, st := NewFakeAsks(), newSteps()
a := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(),
Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40),
Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}}
raw, eerr := sdkstep.EncodeStepAsk(a)
if eerr != nil {
t.Fatal(eerr)
}
src.Publish(sdkstep.AskCheck, raw)
var mu sync.Mutex
var ended []sdkstep.StepEnded
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
Say: func(event string, body any, id string) error {
if event == "step-ended" {
var e sdkstep.StepEnded
b, _ := json.Marshal(body)
if json.Unmarshal(b, &e) == nil {
mu.Lock()
ended = append(ended, e)
mu.Unlock()
}
}
return nil
},
Run: func(context.Context, sdkstep.StepAsk) (any, error) {
return nil, &Declining{Text: "this node builds with another toolchain", Lacks: []string{"the toolchain"}}
}}
if c == "dropped" {
drop, _ := json.Marshal(nodebuildgo.Drop{V: int64(sdkstep.Version), By: "mesh-delivery-builder", Why: "superseded", At: ptr(time.Now())})
p.Load = func(key string) ([]byte, error) {
if key == sdkstep.DropKey(a.ID) {
return drop, nil
}
return nil, nil
}
}
if c == "declined" {
// The decoder refuses needs on a pipeline step ask, so no ask of the queue reaches the decline; the line is held
// by calling it with the ask built whole.
acked := 0
p.decline(&Pulled{Ack: func() error { acked++; return nil }, Term: func() error { return nil }, Nak: func() error { return nil }}, a, []string{"node-build-node"})
if acked != 1 {
t.Errorf("acked %d times, want once", acked)
}
src.Acked = append(src.Acked, "declined")
} else {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go p.Pull(ctx)
}
waitFor(t, "the end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) >= 1 })
waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 })
mu.Lock()
e := ended[0]
mu.Unlock()
if e.Outcome != sdkstep.OutcomeCouldNotRun || e.Pipeline != "main.12.m" {
t.Errorf("the end said %+v, want could not run naming its pipeline", e)
}
if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 {
t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked)
}
})
}
}
// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the
// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after 4 minutes), not 3.
func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) {
src, st := NewFakeAsks(), newSteps()
src.Delivered = 3
a := testAsk(sdkstep.AskDelivery)
_ = src.PublishAsk(a)
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
Prior: func(string) int { return 1 },
Run: func(context.Context, sdkstep.StepAsk) (any, error) {
return nil, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}
}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go p.Pull(ctx)
waitFor(t, "the ask given again", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) == 1 })
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
s, _ := st.get(key)
src.mu.Lock()
delays := append([]time.Duration(nil), src.Delays...)
src.mu.Unlock()
if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != sdkstep.AgainAfter(2) {
t.Errorf("step %+v, delays %v, want again in run 2 after %s", s, delays, sdkstep.AgainAfter(2))
}
}
// Puller.Prior over a steps state's read: the Run of the step the key holds, 0 for an absent key, a failed read or what is no
// step, and nil for no read.
func TestPriorRunsReadsTheRunOfTheStepAKeyHolds(t *testing.T) {
read := func(key string) (json.RawMessage, error) {
switch key {
case "running":
return json.RawMessage(`{"id":"x","run":2}`), nil
case "junk":
return json.RawMessage(`[`), nil
case "broken":
return nil, errors.New("no bus")
}
return nil, nil
}
prior := PriorRuns(read)
for key, want := range map[string]int{"running": 2, "absent": 0, "junk": 0, "broken": 0} {
if got := prior(key); got != want {
t.Errorf("PriorRuns(%s) = %d, want %d", key, got, want)
}
}
if PriorRuns(nil) != nil {
t.Error("PriorRuns(nil) is not nil")
}
}
@@ -244,7 +244,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
go p.Pull(ctx) go p.Pull(ctx)
waitFor(t, "three termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 3 }) waitFor(t, "three acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 3 })
if ran { if ran {
t.Fatal("a wrong or a declined ask ran") t.Fatal("a wrong or a declined ask ran")
} }
@@ -269,7 +269,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) {
return nil, &Declining{Text: "the ask is for 26.13.0 and this node builds with 26.12.1", Lacks: []string{"the agreed toolchain 26.13.0 (recipe 3)"}} return nil, &Declining{Text: "the ask is for 26.13.0 and this node builds with 26.12.1", Lacks: []string{"the agreed toolchain 26.13.0 (recipe 3)"}}
}} }}
go s2.Pull(ctx) go s2.Pull(ctx)
waitFor(t, "the ask termed on the node that sits out", func() bool { _, tm, _ := sit.Fates(); return len(tm) == 1 }) waitFor(t, "the ask acknowledged on the node that sits out", func() bool { ak, _, _ := sit.Fates(); return len(ak) == 1 })
sk, _ := sdkstep.AskStepKey("", "ace", toolchain.ID) sk, _ := sdkstep.AskStepKey("", "ace", toolchain.ID)
ss, _ := sitSteps.get(sk) ss, _ := sitSteps.get(sk)
if d, ok := sdkstep.DeclinedOf(ss); !ok || len(d.Lacks) != 1 || !strings.Contains(ss.Reason, "26.12.1") { if d, ok := sdkstep.DeclinedOf(ss); !ok || len(d.Lacks) != 1 || !strings.Contains(ss.Reason, "26.12.1") {
@@ -538,14 +538,14 @@ func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
go p.Pull(ctx) go p.Pull(ctx)
waitFor(t, "the ask termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 1 }) waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 })
key, _ := sdkstep.AskStepKey("", "shanks", a.ID) key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
s, ok := st.get(key) s, ok := st.get(key)
if ran || !ok || s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") { if ran || !ok || s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") {
t.Fatalf("ran %v, step %+v", ran, s) t.Fatalf("ran %v, step %+v", ran, s)
} }
if acked, _, naked := src.Fates(); len(acked)+len(naked) != 0 { if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 { // A-489: its end written, it leaves the queue by ack
t.Errorf("acked %v naked %v, want it termed alone", acked, naked) t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked)
} }
// ADR 0664 §1 (issue 1748): a run that ends on a transport fault below AskRuns is written again, with its run and when the // ADR 0664 §1 (issue 1748): a run that ends on a transport fault below AskRuns is written again, with its run and when the
// bus gives it, and handed back with that delay, not acknowledged; the AskRuns-th ends failed, passing; a fault of the // bus gives it, and handed back with that delay, not acknowledged; the AskRuns-th ends failed, passing; a fault of the
@@ -620,7 +620,7 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
go p.Pull(ctx) go p.Pull(ctx)
waitFor(t, "both termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 }) waitFor(t, "both acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 2 })
key, _ := sdkstep.AskStepKey("", "ace", good.ID) key, _ := sdkstep.AskStepKey("", "ace", good.ID)
s, ok := st.get(key) s, ok := st.get(key)
if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") { if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") {