steprun: only runs that started count toward poison; an ended ask is acknowledged and its poison end said (A-489)
The bus counts an ask given back before it ran as a delivery, so four asks were ended 'poison' having never run. Puller.Prior (PriorRuns) counts the runs the step key says started. A poisoned pipeline step now says its step-ended, so the runner stops waiting. refuseUnread, refuseAsker and the poison path acknowledge the ask once its end is written, not terminate it. Builds: design 47 G §24, §25; lead answer A-489 (G-22-lead-answers-2.md) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -107,16 +107,37 @@ func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) {
|
||||
return
|
||||
}
|
||||
p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why)
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeFailed, why)
|
||||
_ = msg.Ack() // A-489: ended and written, so it leaves the work queue
|
||||
}
|
||||
|
||||
// sayAskEnded says the end of an ask this holder ends without running it: its step-ended, an ask outside a pipeline as
|
||||
// sayEnded says it, a pipeline step as its own form (design 47 G §602), so the runner that asked hears it.
|
||||
func (p *Puller) sayAskEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) {
|
||||
if ask.Step == nil {
|
||||
p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why, nil, "")
|
||||
} else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil {
|
||||
p.sayEnded(msg, ask, outcome, said, nil, "")
|
||||
} else if end, err := sdkstep.EndOf(ask, p.Node, outcome); err == nil {
|
||||
// A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602).
|
||||
end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask)
|
||||
end.Said, end.At, end.Asker = said, now().UTC(), askerOf(msg, ask)
|
||||
if body, err := sdkstep.EncodeStepEnded(end); err == nil {
|
||||
p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID)
|
||||
}
|
||||
}
|
||||
_ = msg.Term()
|
||||
}
|
||||
|
||||
// PriorRuns is Puller.Prior over a steps state's get (buildrun.Scan.Get): the Run of the step a key holds.
|
||||
func PriorRuns(get func(key string) (json.RawMessage, uint64, error)) func(string) int {
|
||||
return func(key string) int {
|
||||
raw, _, err := get(key)
|
||||
if err != nil || raw == nil {
|
||||
return 0
|
||||
}
|
||||
var s sdkstep.Step
|
||||
if json.Unmarshal(raw, &s) != nil {
|
||||
return 0
|
||||
}
|
||||
return s.Run
|
||||
}
|
||||
}
|
||||
|
||||
// giveBack hands an ask back to be judged again a little later, once the seat list says more. Why it cannot be judged is
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) {
|
||||
key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID)
|
||||
waitFor(t, "the step keyed "+key, func() bool { _, ok := st.get(key); return ok })
|
||||
case "refused":
|
||||
waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 })
|
||||
waitFor(t, "the ask ended", func() bool { acked, _, _ := src.Fates(); return len(acked) == 1 }) // A-489: ended, written, acknowledged
|
||||
key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID)
|
||||
s, _ := st.get(key)
|
||||
if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) {
|
||||
|
||||
@@ -87,7 +87,11 @@ type Puller struct {
|
||||
KeepOutput func(object string, body []byte) error
|
||||
// Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions);
|
||||
// found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163).
|
||||
Callers func(kind string) (callers []string, found bool)
|
||||
Callers func(kind string) (callers []string, found bool)
|
||||
// Prior says how many runs of the ask under a step key started before (the Run of the step the key holds, PriorRuns; 0 for
|
||||
// none), so the poison rule and the transport fault's run count only runs that started, never an ask given back before it
|
||||
// ran. Nil counts the bus's deliveries (a test's).
|
||||
Prior func(key string) int
|
||||
ProgressEvery time.Duration
|
||||
Logf func(format string, args ...any)
|
||||
// Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull
|
||||
@@ -628,7 +632,7 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) {
|
||||
p.logf("the unread ask %s cannot be answered on %s: %v", probe.ID, p.Node, werr)
|
||||
} else {
|
||||
p.logf("an ask refused: %v", why)
|
||||
_ = msg.Term()
|
||||
_ = msg.Ack() // A-489: ended and written, so it leaves the work queue
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -673,19 +677,28 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
p.decline(msg, ask, lacks)
|
||||
return
|
||||
}
|
||||
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), int(msg.Delivered))
|
||||
// The run this is: the runs of the ask that started before it, from its step key, and one more. The bus's delivery count
|
||||
// also counts an ask given back before it ran (a seat list not read yet, a start that failed), which is no run.
|
||||
prior := int(msg.Delivered) - 1
|
||||
if p.Prior != nil {
|
||||
if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil {
|
||||
prior = p.Prior(akey)
|
||||
}
|
||||
}
|
||||
step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1)
|
||||
if err != nil {
|
||||
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if msg.Delivered > sdkstep.AskRuns {
|
||||
if prior >= sdkstep.AskRuns {
|
||||
// Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said
|
||||
// (issue 1907): the asker's key fails with why.
|
||||
why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", msg.Delivered-1)
|
||||
why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior)
|
||||
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
|
||||
_, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then
|
||||
_ = msg.Term()
|
||||
_, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then
|
||||
p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error()) // and its asker, a pipeline's runner too, hears it
|
||||
_ = msg.Ack() // A-489
|
||||
return
|
||||
}
|
||||
module := ""
|
||||
@@ -775,7 +788,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
|
||||
// The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs.
|
||||
return
|
||||
}
|
||||
if run := int(msg.Delivered); err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil {
|
||||
if run := step.Run; err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil {
|
||||
// A transport fault (ADR 0664 §1): the step says it is given again and the bus gives it after a delay, to this node or
|
||||
// another; it is not acknowledged, so its asker sees no end. Run AskRuns times it ends failed, said as a transport fault.
|
||||
delay := sdkstep.AgainAfter(run)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package steprun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// Design 47 G *Asked on a work queue, ended by an event* (§24): the bus counts an ask given back before it ran (a seat list
|
||||
// not read yet) as a delivery, and that is no run. Only runs that started count toward the poison rule (Puller.Prior, from the
|
||||
// step key), and a poisoned pipeline step is ended by its step-ended event, so the runner that asked hears it.
|
||||
func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) {
|
||||
stepAsk := func() sdkstep.StepAsk {
|
||||
return sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(),
|
||||
Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40),
|
||||
Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
delivered uint64
|
||||
prior int
|
||||
poisoned bool
|
||||
}{
|
||||
{"given back four times, never run", sdkstep.AskRuns + 2, 0, false},
|
||||
{"three runs started and none ended", 3, sdkstep.AskRuns, true},
|
||||
{"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = c.delivered
|
||||
a := stepAsk()
|
||||
raw, err := sdkstep.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src.Publish(sdkstep.AskCheck, raw) // a pipeline step rides the check queue
|
||||
var mu sync.Mutex
|
||||
var ended []sdkstep.StepEnded
|
||||
ran := make(chan struct{}, 1)
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Prior: func(string) int { return c.prior },
|
||||
Say: func(event string, body any, id string) error {
|
||||
if event == "step-ended" {
|
||||
var e sdkstep.StepEnded
|
||||
raw, _ := json.Marshal(body)
|
||||
if err := json.Unmarshal(raw, &e); err == nil {
|
||||
mu.Lock()
|
||||
ended = append(ended, e)
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
if c.poisoned {
|
||||
waitFor(t, "the poisoned ask's end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) == 1 })
|
||||
mu.Lock()
|
||||
e := ended[0]
|
||||
mu.Unlock()
|
||||
if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, "poison") || e.Pipeline != "main.12.m" {
|
||||
t.Fatalf("the end said %+v, want could not run, poison, naming its pipeline", e)
|
||||
}
|
||||
select {
|
||||
case <-ran:
|
||||
t.Fatal("a poisoned ask ran")
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ran:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("an ask only given back was ended as poison, and never ran")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -244,7 +244,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "three termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 3 })
|
||||
waitFor(t, "three ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 3 })
|
||||
if ran {
|
||||
t.Fatal("a wrong or a declined ask ran")
|
||||
}
|
||||
@@ -538,14 +538,14 @@ func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 1 })
|
||||
waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 })
|
||||
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
|
||||
s, ok := st.get(key)
|
||||
if ran || !ok || s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") {
|
||||
t.Fatalf("ran %v, step %+v", ran, s)
|
||||
}
|
||||
if acked, _, naked := src.Fates(); len(acked)+len(naked) != 0 {
|
||||
t.Errorf("acked %v naked %v, want it termed alone", acked, naked)
|
||||
if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 { // A-489: its end written, it leaves the queue by ack
|
||||
t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked)
|
||||
}
|
||||
// ADR 0664 §1 (issue 1748): a run that ends on a transport fault below AskRuns is written again, with its run and when the
|
||||
// bus gives it, and handed back with that delay, not acknowledged; the AskRuns-th ends failed, passing; a fault of the
|
||||
@@ -620,7 +620,7 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "both termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 })
|
||||
waitFor(t, "both ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 2 })
|
||||
key, _ := sdkstep.AskStepKey("", "ace", good.ID)
|
||||
s, ok := st.get(key)
|
||||
if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") {
|
||||
|
||||
Reference in New Issue
Block a user