From 79bca88c8a2517ead1ff3a434bac4e8346413582 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 15:23:13 +0100 Subject: [PATCH 1/7] steprun: a build module whose steps cannot say a step's end fails its own test (conformance.StepEnds) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A step ask refused live with 'this build module says no step's end' (mesh-build-node set no SayStep). AskSteps.SaysItsEnds and the kit case conformance.StepEnds refuse that where the module is built. Builds: design 47 G §24 (Asked on a work queue, ended by an event) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- modules/mesh-delivery-sdk/steprun/askrun.go | 15 +++++++++++++++ .../steprun/conformance/conformance.go | 11 +++++++++++ .../steprun/conformance/conformance_test.go | 15 +++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/modules/mesh-delivery-sdk/steprun/askrun.go b/modules/mesh-delivery-sdk/steprun/askrun.go index 935ad7b6..93f18ec5 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun.go +++ b/modules/mesh-delivery-sdk/steprun/askrun.go @@ -3,6 +3,7 @@ package steprun import ( "context" "encoding/json" + "errors" "fmt" "maps" "path" @@ -56,6 +57,20 @@ type AskSteps struct { SayStep func(end sdkstep.StepEnded) error } +// SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to +// publish it (design 47 G *Asked on a work queue, ended by an event*, §24). A build module whose steps cannot is refused where +// it is built (conformance.StepEnds, which every build module's test runs, and the catalogue check failurekit), never +// discovered live as a step ask refused with "says no step's end". +func (s AskSteps) SaysItsEnds() error { + switch { + case s.SayStep == nil: + return errors.New("its AskSteps has no SayStep, so a step ask is refused: set it to SayStepOn the seat's language and node") + case s.Node == "": + return errors.New("its AskSteps names no Node, so the end of a step ask names no node") + } + return nil +} + // RunAsk runs ask's job with steps and answers a sdkstep.BuildAnswer or a sdkstep.CheckAnswer. func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, error) { if ask.Build != nil { diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go index f04eadbb..05d3a4f1 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go @@ -12,6 +12,7 @@ package conformance import ( "embed" "fmt" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "slices" "strings" "testing" @@ -277,3 +278,13 @@ func judgePassing(c PassingCase, failed Failed) string { } return "" } + +// StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end +// (steprun.AskSteps.SaysItsEnds), or the module would refuse every step ask the runner sends, as mesh-build-node did on its +// first live pipeline (design 47 G §24). A catalogue check (failurekit) refuses a build module whose tests do not call it. +func StepEnds(t *testing.T, steps steprun.AskSteps) { + t.Helper() + if err := steps.SaysItsEnds(); err != nil { + t.Errorf("the build module's steps cannot say a step's end: %v", err) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go index 159c5acc..e24f65bd 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go @@ -181,3 +181,18 @@ func TestTheFailureContract(t *testing.T) { } }) } + +// A build module whose steps cannot say a step's end fails the case; one that can passes. +func TestStepEndsRefusesStepsThatCannotSayTheirEnd(t *testing.T) { + say := func(sdkstep.StepEnded) error { return nil } + for _, c := range []struct { + name string + s steprun.AskSteps + bad bool + }{{"no SayStep", steprun.AskSteps{Node: "n"}, true}, {"no node", steprun.AskSteps{SayStep: say}, true}, {"both", steprun.AskSteps{Node: "n", SayStep: say}, false}} { + if (c.s.SaysItsEnds() != nil) != c.bad { + t.Errorf("%s: SaysItsEnds = %v, want refused %v", c.name, c.s.SaysItsEnds(), c.bad) + } + } + StepEnds(t, steprun.AskSteps{Node: "n", SayStep: say}) +} From 78adc23d7013e8ade845ee54d724deac4539f98d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 15:23:13 +0100 Subject: [PATCH 2/7] steprun: only runs that started count toward poison; an ended ask is acknowledged and its poison end said (A-489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bus counts an ask given back before it ran as a delivery, so four asks were ended 'poison' having never run. Puller.Prior (PriorRuns) counts the runs the step key says started. A poisoned pipeline step now says its step-ended, so the runner stops waiting. refuseUnread, refuseAsker and the poison path acknowledge the ask once its end is written, not terminate it. Builds: design 47 G §24, §25; lead answer A-489 (G-22-lead-answers-2.md) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- modules/mesh-delivery-sdk/steprun/askers.go | 29 ++++++- .../mesh-delivery-sdk/steprun/askers_test.go | 2 +- modules/mesh-delivery-sdk/steprun/pull.go | 29 +++++-- .../steprun/pull_poison_test.go | 85 +++++++++++++++++++ .../mesh-delivery-sdk/steprun/pull_test.go | 10 +-- 5 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery-sdk/steprun/pull_poison_test.go diff --git a/modules/mesh-delivery-sdk/steprun/askers.go b/modules/mesh-delivery-sdk/steprun/askers.go index 5013765a..b1319894 100644 --- a/modules/mesh-delivery-sdk/steprun/askers.go +++ b/modules/mesh-delivery-sdk/steprun/askers.go @@ -107,16 +107,37 @@ func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) { return } p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why) + p.sayAskEnded(msg, ask, sdkstep.OutcomeFailed, why) + _ = msg.Ack() // A-489: ended and written, so it leaves the work queue +} + +// sayAskEnded says the end of an ask this holder ends without running it: its step-ended, an ask outside a pipeline as +// sayEnded says it, a pipeline step as its own form (design 47 G §602), so the runner that asked hears it. +func (p *Puller) sayAskEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) { if ask.Step == nil { - p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why, nil, "") - } else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil { + p.sayEnded(msg, ask, outcome, said, nil, "") + } else if end, err := sdkstep.EndOf(ask, p.Node, outcome); err == nil { // A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602). - end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask) + end.Said, end.At, end.Asker = said, now().UTC(), askerOf(msg, ask) if body, err := sdkstep.EncodeStepEnded(end); err == nil { p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) } } - _ = msg.Term() +} + +// PriorRuns is Puller.Prior over a steps state's get (buildrun.Scan.Get): the Run of the step a key holds. +func PriorRuns(get func(key string) (json.RawMessage, uint64, error)) func(string) int { + return func(key string) int { + raw, _, err := get(key) + if err != nil || raw == nil { + return 0 + } + var s sdkstep.Step + if json.Unmarshal(raw, &s) != nil { + return 0 + } + return s.Run + } } // giveBack hands an ask back to be judged again a little later, once the seat list says more. Why it cannot be judged is diff --git a/modules/mesh-delivery-sdk/steprun/askers_test.go b/modules/mesh-delivery-sdk/steprun/askers_test.go index f928ccd3..eac24966 100644 --- a/modules/mesh-delivery-sdk/steprun/askers_test.go +++ b/modules/mesh-delivery-sdk/steprun/askers_test.go @@ -65,7 +65,7 @@ func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) { key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID) waitFor(t, "the step keyed "+key, func() bool { _, ok := st.get(key); return ok }) case "refused": - waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 }) + waitFor(t, "the ask ended", func() bool { acked, _, _ := src.Fates(); return len(acked) == 1 }) // A-489: ended, written, acknowledged key, _ := sdkstep.AskStepKey(asker, "shanks", c.ask.ID) s, _ := st.get(key) if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) { diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 88d2cd87..2bc1b2ed 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -87,7 +87,11 @@ type Puller struct { KeepOutput func(object string, body []byte) error // Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions); // found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163). - Callers func(kind string) (callers []string, found bool) + Callers func(kind string) (callers []string, found bool) + // Prior says how many runs of the ask under a step key started before (the Run of the step the key holds, PriorRuns; 0 for + // none), so the poison rule and the transport fault's run count only runs that started, never an ask given back before it + // ran. Nil counts the bus's deliveries (a test's). + Prior func(key string) int ProgressEvery time.Duration Logf func(format string, args ...any) // Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull @@ -628,7 +632,7 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) { p.logf("the unread ask %s cannot be answered on %s: %v", probe.ID, p.Node, werr) } else { p.logf("an ask refused: %v", why) - _ = msg.Term() + _ = msg.Ack() // A-489: ended and written, so it leaves the work queue return } } @@ -673,19 +677,28 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.decline(msg, ask, lacks) return } - step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), int(msg.Delivered)) + // The run this is: the runs of the ask that started before it, from its step key, and one more. The bus's delivery count + // also counts an ask given back before it ran (a seat list not read yet, a start that failed), which is no run. + prior := int(msg.Delivered) - 1 + if p.Prior != nil { + if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil { + prior = p.Prior(akey) + } + } + step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1) if err != nil { p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err) _ = msg.Nak() return } - if msg.Delivered > sdkstep.AskRuns { + if prior >= sdkstep.AskRuns { // Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said // (issue 1907): the asker's key fails with why. - why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", msg.Delivered-1) + why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior) p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why) - _, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then - _ = msg.Term() + _, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error()) // and its asker, a pipeline's runner too, hears it + _ = msg.Ack() // A-489 return } module := "" @@ -775,7 +788,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key // The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs. return } - if run := int(msg.Delivered); err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil { + if run := step.Run; err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil { // A transport fault (ADR 0664 §1): the step says it is given again and the bus gives it after a delay, to this node or // another; it is not acknowledged, so its asker sees no end. Run AskRuns times it ends failed, said as a transport fault. delay := sdkstep.AgainAfter(run) diff --git a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go new file mode 100644 index 00000000..b39697fc --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go @@ -0,0 +1,85 @@ +package steprun + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" +) + +// Design 47 G *Asked on a work queue, ended by an event* (§24): the bus counts an ask given back before it ran (a seat list +// not read yet) as a delivery, and that is no run. Only runs that started count toward the poison rule (Puller.Prior, from the +// step key), and a poisoned pipeline step is ended by its step-ended event, so the runner that asked hears it. +func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { + stepAsk := func() sdkstep.StepAsk { + return sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(), + Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40), + Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}} + } + for _, c := range []struct { + name string + delivered uint64 + prior int + poisoned bool + }{ + {"given back four times, never run", sdkstep.AskRuns + 2, 0, false}, + {"three runs started and none ended", 3, sdkstep.AskRuns, true}, + {"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, true}, + } { + t.Run(c.name, func(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + src.Delivered = c.delivered + a := stepAsk() + raw, err := sdkstep.EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + src.Publish(sdkstep.AskCheck, raw) // a pipeline step rides the check queue + var mu sync.Mutex + var ended []sdkstep.StepEnded + ran := make(chan struct{}, 1) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, + PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Prior: func(string) int { return c.prior }, + Say: func(event string, body any, id string) error { + if event == "step-ended" { + var e sdkstep.StepEnded + raw, _ := json.Marshal(body) + if err := json.Unmarshal(raw, &e); err == nil { + mu.Lock() + ended = append(ended, e) + mu.Unlock() + } + } + return nil + }, + Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + if c.poisoned { + waitFor(t, "the poisoned ask's end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) == 1 }) + mu.Lock() + e := ended[0] + mu.Unlock() + if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, "poison") || e.Pipeline != "main.12.m" { + t.Fatalf("the end said %+v, want could not run, poison, naming its pipeline", e) + } + select { + case <-ran: + t.Fatal("a poisoned ask ran") + default: + } + return + } + select { + case <-ran: + case <-time.After(5 * time.Second): + t.Fatal("an ask only given back was ended as poison, and never ran") + } + }) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_test.go b/modules/mesh-delivery-sdk/steprun/pull_test.go index 5fc90720..d5dd4d0d 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_test.go @@ -244,7 +244,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "three termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 3 }) + waitFor(t, "three ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 3 }) if ran { t.Fatal("a wrong or a declined ask ran") } @@ -538,14 +538,14 @@ func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "the ask termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 1 }) + waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 }) key, _ := sdkstep.AskStepKey("", "shanks", a.ID) s, ok := st.get(key) if ran || !ok || s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") { t.Fatalf("ran %v, step %+v", ran, s) } - if acked, _, naked := src.Fates(); len(acked)+len(naked) != 0 { - t.Errorf("acked %v naked %v, want it termed alone", acked, naked) + if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 { // A-489: its end written, it leaves the queue by ack + t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked) } // ADR 0664 §1 (issue 1748): a run that ends on a transport fault below AskRuns is written again, with its run and when the // bus gives it, and handed back with that delay, not acknowledged; the AskRuns-th ends failed, passing; a fault of the @@ -620,7 +620,7 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "both termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 }) + waitFor(t, "both ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 2 }) key, _ := sdkstep.AskStepKey("", "ace", good.ID) s, ok := st.get(key) if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") { From 16feb56fd4b6ac945aaff19b5b99b3513c2cb086 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 15:24:37 +0100 Subject: [PATCH 3/7] steprun: PriorRuns reads a module's ReadSteps shape Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- modules/mesh-delivery-sdk/steprun/askers.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/askers.go b/modules/mesh-delivery-sdk/steprun/askers.go index b1319894..2f8caa64 100644 --- a/modules/mesh-delivery-sdk/steprun/askers.go +++ b/modules/mesh-delivery-sdk/steprun/askers.go @@ -125,10 +125,14 @@ func (p *Puller) sayAskEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said str } } -// PriorRuns is Puller.Prior over a steps state's get (buildrun.Scan.Get): the Run of the step a key holds. -func PriorRuns(get func(key string) (json.RawMessage, uint64, error)) func(string) int { +// PriorRuns is Puller.Prior over a steps state's read (a module's ReadSteps.Get): the Run of the step a key holds. Nil for no +// read (a test's), so the bus's delivery count stands. +func PriorRuns(get func(key string) (json.RawMessage, error)) func(string) int { + if get == nil { + return nil + } return func(key string) int { - raw, _, err := get(key) + raw, err := get(key) if err != nil || raw == nil { return 0 } From 92196dde3a33448d222cadfe4d99f259c4afa8ea Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 15:38:16 +0100 Subject: [PATCH 4/7] steprun: an ask on its last delivery is ended, every end is acknowledged, an unwritten end is given back (A-489a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §24, §25; lead answers A-489, A-489a Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- modules/mesh-delivery-sdk/steprun/askrun.go | 2 +- .../steprun/conformance/conformance.go | 2 +- modules/mesh-delivery-sdk/steprun/pull.go | 73 ++++++++++----- modules/mesh-delivery-sdk/steprun/pull_bus.go | 6 +- .../mesh-delivery-sdk/steprun/pull_drop.go | 3 +- .../steprun/pull_poison_test.go | 88 +++++++++++++++++-- .../mesh-delivery-sdk/steprun/pull_test.go | 6 +- 7 files changed, 144 insertions(+), 36 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/askrun.go b/modules/mesh-delivery-sdk/steprun/askrun.go index 93f18ec5..01525731 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun.go +++ b/modules/mesh-delivery-sdk/steprun/askrun.go @@ -57,7 +57,7 @@ type AskSteps struct { SayStep func(end sdkstep.StepEnded) error } -// SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to +// A-489b (G-22-lead-answers-2.md): SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to // publish it (design 47 G *Asked on a work queue, ended by an event*, §24). A build module whose steps cannot is refused where // it is built (conformance.StepEnds, which every build module's test runs, and the catalogue check failurekit), never // discovered live as a step ask refused with "says no step's end". diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go index 05d3a4f1..a8eb5b33 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go @@ -279,7 +279,7 @@ func judgePassing(c PassingCase, failed Failed) string { return "" } -// StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end +// A-489b: StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end // (steprun.AskSteps.SaysItsEnds), or the module would refuse every step ask the runner sends, as mesh-build-node did on its // first live pipeline (design 47 G §24). A catalogue check (failurekit) refuses a build module whose tests do not call it. func StepEnds(t *testing.T, steps steprun.AskSteps) { diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 2bc1b2ed..15418a1a 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -32,6 +32,8 @@ type Pulled struct { Subject string // Delivered is how many times the bus has handed it over, this time included. Delivered uint64 + // Sequence is its sequence in the stream (0 for a source that has none), which the journal names for an ask that cannot be read. + Sequence uint64 // Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its // acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it. Ack, InProgress, Nak, Term func() error @@ -580,7 +582,8 @@ func (p *Puller) decline(msg *Pulled, ask sdkstep.StepAsk, lacks []string) { return } p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", ")) - _ = msg.Term() + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "this node lacks "+strings.Join(lacks, ", ")) + _ = msg.Ack() // A-489a: its end is written p.mu.Lock() if p.declined == nil { p.declined = map[string]bool{} @@ -602,7 +605,7 @@ func (p *Puller) declineRun(msg *Pulled, ask sdkstep.StepAsk, key string, step s return } p.logf("the ask %s is declined on %s: %s", ask.ID, p.Node, d.Text) - _ = msg.Term() + _ = msg.Ack() // A-489a: its end is written (run() says it) p.mu.Lock() if p.declined == nil { p.declined = map[string]bool{} @@ -637,8 +640,8 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) { } } } - p.logf("an ask dropped: %v", why) - _ = msg.Term() + p.logf("an ask dropped, no id to answer it by, acknowledged with this line as its only record (stream sequence %d): %v", msg.Sequence, why) + _ = msg.Ack() // A-489a } func withKind(msg *Pulled, kind string) *Pulled { @@ -648,6 +651,37 @@ func withKind(msg *Pulled, kind string) *Pulled { return msg } +// priorRuns is how many runs of the ask started before this delivery: its step key's (Puller.Prior), else the bus's delivery +// count less this one. The bus's count also counts an ask given back before it ran, which is no run. +func (p *Puller) priorRuns(msg *Pulled, ask sdkstep.StepAsk) int { + if p.Prior != nil { + if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil { + return p.Prior(akey) + } + } + return int(msg.Delivered) - 1 +} + +// endUnrun ends an ask that is run no more (A-489, A-489a): poison when runs of it started and none ended, out of its +// deliveries when none did. Its step ends and its step-ended is said, and only then it is acknowledged; an end that could not +// be written gives it back, never acknowledged. +func (p *Puller) endUnrun(msg *Pulled, ask sdkstep.StepAsk, key string, step sdkstep.Step, prior int) { + var why error + if prior > 0 { + why = fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior) + } else { + why = fmt.Errorf("the ask is on its last delivery (%d) and no run of it started: it is out of its deliveries and run no more", msg.Delivered) + } + p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why) + if _, ferr := finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf); ferr != nil { + p.logf("the end of the ask %s could not be written on %s, so it is given back: %v", ask.ID, p.Node, ferr) + _ = msg.Nak() + return + } + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error()) + _ = msg.Ack() +} + // take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id. func (p *Puller) take(ctx context.Context, msg *Pulled) { kind := msg.Kind @@ -662,6 +696,18 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.refuseUnread(msg, err) return } + if msg.Delivered >= sdkstep.AskMaxDeliver { + // Its last delivery (A-489a): the bus gives it no more, so it is ended here, run or not, never given back. + prior := p.priorRuns(msg, ask) + step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1) + if err != nil { + p.logf("the ask %s is on its last delivery and cannot start its end on %s: %v", ask.ID, p.Node, err) + _ = msg.Nak() + return + } + p.endUnrun(msg, ask, key, step, prior) + return + } if why, judged := p.askerRefusal(msg, ask); !judged { p.giveBack(msg, ask) return @@ -677,14 +723,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.decline(msg, ask, lacks) return } - // The run this is: the runs of the ask that started before it, from its step key, and one more. The bus's delivery count - // also counts an ask given back before it ran (a seat list not read yet, a start that failed), which is no run. - prior := int(msg.Delivered) - 1 - if p.Prior != nil { - if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil { - prior = p.Prior(akey) - } - } + prior := p.priorRuns(msg, ask) step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1) if err != nil { p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err) @@ -692,13 +731,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { return } if prior >= sdkstep.AskRuns { - // Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said - // (issue 1907): the asker's key fails with why. - why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior) - p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why) - _, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then - p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error()) // and its asker, a pipeline's runner too, hears it - _ = msg.Ack() // A-489 + p.endUnrun(msg, ask, key, step, prior) return } module := "" @@ -781,7 +814,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key var dec *Declining if errors.As(err, &dec) && ctx.Err() == nil { p.declineRun(msg, ask, key, step, dec) - p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text, nil, "") + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text) return } if ctx.Err() != nil { diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index 5669ec4b..08e9a1b8 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -116,11 +116,11 @@ func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait ti if !ok { return nil, batch.Error() } - var delivered uint64 = 1 + var delivered, sequence uint64 = 1, 0 if meta, err := msg.Metadata(); err == nil && meta != nil { - delivered = meta.NumDelivered + delivered, sequence = meta.NumDelivered, meta.Sequence.Stream } - return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, + return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, Sequence: sequence, Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term, NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil case <-ctx.Done(): diff --git a/modules/mesh-delivery-sdk/steprun/pull_drop.go b/modules/mesh-delivery-sdk/steprun/pull_drop.go index 96d82825..af837dbd 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_drop.go +++ b/modules/mesh-delivery-sdk/steprun/pull_drop.go @@ -147,7 +147,8 @@ func (p *Puller) refuseDropped(msg *Pulled, ask sdkstep.StepAsk, d *nodebuildgo. return } p.logf("the ask %s is refused on %s: it was dropped (%s)", ask.ID, p.Node, d.Why) - _ = msg.Term() + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "it was dropped: "+d.Why) + _ = msg.Ack() // A-489a: its end is written p.forgetDrop(ask.ID) } diff --git a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go index b39697fc..bfa165f8 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go @@ -3,6 +3,7 @@ package steprun import ( "context" "encoding/json" + "errors" "strings" "sync" "testing" @@ -24,11 +25,13 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { name string delivered uint64 prior int - poisoned bool + ends string // what its end says; "" when it must run }{ - {"given back four times, never run", sdkstep.AskRuns + 2, 0, false}, - {"three runs started and none ended", 3, sdkstep.AskRuns, true}, - {"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, true}, + {"given back three times, never run", sdkstep.AskRuns + 1, 0, ""}, + {"three runs started and none ended", 3, sdkstep.AskRuns, "poison"}, + {"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, "poison"}, + {"its last delivery, given back before and never run", sdkstep.AskMaxDeliver, 0, "out of its deliveries"}, + {"its last delivery after a run that started", sdkstep.AskMaxDeliver, 1, "poison"}, } { t.Run(c.name, func(t *testing.T) { src, st := NewFakeAsks(), newSteps() @@ -60,17 +63,20 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - if c.poisoned { + if c.ends != "" { waitFor(t, "the poisoned ask's end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) == 1 }) mu.Lock() e := ended[0] mu.Unlock() - if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, "poison") || e.Pipeline != "main.12.m" { + if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, c.ends) || e.Pipeline != "main.12.m" { t.Fatalf("the end said %+v, want could not run, poison, naming its pipeline", e) } + if acked, termed, naked := src.Fates(); len(acked) != 1 || len(termed)+len(naked) != 0 { + t.Errorf("acked %v termed %v naked %v, want it acknowledged once its end is written (A-489a)", acked, termed, naked) + } select { case <-ran: - t.Fatal("a poisoned ask ran") + t.Fatal("an ended ask ran") default: } return @@ -83,3 +89,71 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { }) } } + +// A-489a: an end that could not be written is given back, never acknowledged. +func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) { + src := NewFakeAsks() + src.Delivered = sdkstep.AskMaxDeliver + _ = src.PublishAsk(testAsk(sdkstep.AskDelivery)) + put := func(string, any) (uint64, error) { return 0, errors.New("the state is full") } + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the ask given back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 }) + if acked, _, _ := src.Fates(); len(acked) != 0 { + t.Errorf("acked %v, want none: no end was written", acked) + } +} + +// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the +// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after 4 minutes), not 3. +func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + src.Delivered = 3 + a := testAsk(sdkstep.AskDelivery) + _ = src.PublishAsk(a) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Prior: func(string) int { return 1 }, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { + return nil, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")} + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the ask given again", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) == 1 }) + key, _ := sdkstep.AskStepKey("", "shanks", a.ID) + s, _ := st.get(key) + src.mu.Lock() + delays := append([]time.Duration(nil), src.Delays...) + src.mu.Unlock() + if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != sdkstep.AgainAfter(2) { + t.Errorf("step %+v, delays %v, want again in run 2 after %s", s, delays, sdkstep.AgainAfter(2)) + } +} + +// Puller.Prior over a steps state's read: the Run of the step the key holds, 0 for an absent key, a failed read or what is no +// step, and nil for no read. +func TestPriorRunsReadsTheRunOfTheStepAKeyHolds(t *testing.T) { + read := func(key string) (json.RawMessage, error) { + switch key { + case "running": + return json.RawMessage(`{"id":"x","run":2}`), nil + case "junk": + return json.RawMessage(`[`), nil + case "broken": + return nil, errors.New("no bus") + } + return nil, nil + } + prior := PriorRuns(read) + for key, want := range map[string]int{"running": 2, "absent": 0, "junk": 0, "broken": 0} { + if got := prior(key); got != want { + t.Errorf("PriorRuns(%s) = %d, want %d", key, got, want) + } + } + if PriorRuns(nil) != nil { + t.Error("PriorRuns(nil) is not nil") + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_test.go b/modules/mesh-delivery-sdk/steprun/pull_test.go index d5dd4d0d..f5cc9578 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_test.go @@ -244,7 +244,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "three ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 3 }) + waitFor(t, "three acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 3 }) if ran { t.Fatal("a wrong or a declined ask ran") } @@ -269,7 +269,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) { return nil, &Declining{Text: "the ask is for 26.13.0 and this node builds with 26.12.1", Lacks: []string{"the agreed toolchain 26.13.0 (recipe 3)"}} }} go s2.Pull(ctx) - waitFor(t, "the ask termed on the node that sits out", func() bool { _, tm, _ := sit.Fates(); return len(tm) == 1 }) + waitFor(t, "the ask acknowledged on the node that sits out", func() bool { ak, _, _ := sit.Fates(); return len(ak) == 1 }) sk, _ := sdkstep.AskStepKey("", "ace", toolchain.ID) ss, _ := sitSteps.get(sk) if d, ok := sdkstep.DeclinedOf(ss); !ok || len(d.Lacks) != 1 || !strings.Contains(ss.Reason, "26.12.1") { @@ -620,7 +620,7 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "both ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 2 }) + waitFor(t, "both acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 2 }) key, _ := sdkstep.AskStepKey("", "ace", good.ID) s, ok := st.get(key) if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") { From 334bc5476751fc023360e38783d8d29efddc57d6 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 15:42:16 +0100 Subject: [PATCH 5/7] steprun: an unread ask whose refusal cannot be written is given back; the unwritten-end and said-end cases assert what they claim; kit case RunsCounted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §24; lead answers A-489a, A-489b Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../steprun/conformance/conformance.go | 23 ++++ .../steprun/conformance/conformance_test.go | 15 +++ modules/mesh-delivery-sdk/steprun/pull.go | 4 +- modules/mesh-delivery-sdk/steprun/pull_bus.go | 4 +- .../steprun/pull_poison_test.go | 116 +++++++++++++++--- 5 files changed, 145 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go index a8eb5b33..2338afa1 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go @@ -11,6 +11,8 @@ package conformance import ( "embed" + "encoding/json" + "errors" "fmt" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "slices" @@ -288,3 +290,24 @@ func StepEnds(t *testing.T, steps steprun.AskSteps) { t.Errorf("the build module's steps cannot say a step's end: %v", err) } } + +// RunsCounted is the case every build module runs on its puller (A-489a): the poison rule and a transport fault's run count the +// runs the module's steps state says started, so the puller is built with Prior over the module's own read of that state. The +// module builds its puller with read as its steps state's read; a step key it holds with a started run must count as that run. +func RunsCounted(t *testing.T, puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) { + t.Helper() + if err := runsCounted(puller); err != nil { + t.Error(err) + } +} + +func runsCounted(puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) error { + p := puller(func(string) (json.RawMessage, error) { return json.RawMessage(`{"run":2}`), nil }) + switch { + case p == nil || p.Prior == nil: + return errors.New("the build module's puller has no Prior: an ask given back would count toward poison") + case p.Prior("any") != 2: + return fmt.Errorf("Prior over a step key holding run 2 = %d, want 2: the puller does not read the module's steps state", p.Prior("any")) + } + return nil +} diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go index e24f65bd..51c33107 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go @@ -196,3 +196,18 @@ func TestStepEndsRefusesStepsThatCannotSayTheirEnd(t *testing.T) { } StepEnds(t, steprun.AskSteps{Node: "n", SayStep: say}) } + +// A puller built without Prior, or with one that does not read the module's steps state, fails the case. +func TestRunsCountedRefusesAPullerThatDoesNotReadTheStepsState(t *testing.T) { + wired := func(read func(string) (json.RawMessage, error)) *steprun.Puller { + return &steprun.Puller{Prior: steprun.PriorRuns(read)} + } + if err := runsCounted(wired); err != nil { + t.Errorf("a wired puller: %v", err) + } + for name, p := range map[string]*steprun.Puller{"none": {}, "constant": {Prior: func(string) int { return 0 }}} { + if runsCounted(func(func(string) (json.RawMessage, error)) *steprun.Puller { return p }) == nil { + t.Errorf("%s: the case passed a puller that does not count the state's runs", name) + } + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 15418a1a..d753a92b 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -632,7 +632,9 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) { step := sdkstep.Step{ID: probe.ID, Tool: tool, Node: p.Node, State: sdkstep.StepRefused, Started: at, Ended: at, Reason: fmt.Sprintf("this build module on %s cannot read the ask: %v", p.Node, why), Owner: buildrun.Owner} if werr := writeEnd(p.Put, key, step); werr != nil { - p.logf("the unread ask %s cannot be answered on %s: %v", probe.ID, p.Node, werr) + p.logf("the unread ask %s cannot be answered on %s, so it is given back: %v", probe.ID, p.Node, werr) + _ = msg.Nak() // A-489a: an end that could not be written is given back, never acknowledged + return } else { p.logf("an ask refused: %v", why) _ = msg.Ack() // A-489: ended and written, so it leaves the work queue diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index 08e9a1b8..4053993a 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -146,12 +146,12 @@ func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language str subject, _ := sdkstep.AskSubject(language, kind) // Two workers whose filters do not overlap: main's subject and the asker's, `.` (design 47 G §544). if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject}, - AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { + AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: sdkstep.AskMaxDeliver}); err != nil { return fmt.Errorf("the worker %s: %w", worker, err) } named, _ := sdkstep.AskNamedWorker(language, kind) if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"}, - AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { + AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: sdkstep.AskMaxDeliver}); err != nil { return fmt.Errorf("the worker %s: %w", named, err) } } diff --git a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go index bfa165f8..974cb160 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" ) @@ -45,7 +46,7 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { var mu sync.Mutex var ended []sdkstep.StepEnded ran := make(chan struct{}, 1) - p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, + p := &Puller{Source: src, Language: sdkstep.AskGo, Logf: t.Logf, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Prior: func(string) int { return c.prior }, Say: func(event string, body any, id string) error { if event == "step-ended" { @@ -90,20 +91,107 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { } } -// A-489a: an end that could not be written is given back, never acknowledged. +// A-489a: an end that could not be written is given back, never acknowledged: the ask's start is written, its end is not +// (poison), and an unreadable ask with an id whose refusal cannot be written. func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) { - src := NewFakeAsks() - src.Delivered = sdkstep.AskMaxDeliver - _ = src.PublishAsk(testAsk(sdkstep.AskDelivery)) - put := func(string, any) (uint64, error) { return 0, errors.New("the state is full") } - p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, - Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, nil }} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go p.Pull(ctx) - waitFor(t, "the ask given back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 }) - if acked, _, _ := src.Fates(); len(acked) != 0 { - t.Errorf("acked %v, want none: no end was written", acked) + was := EndAttempts + EndAttempts = 1 + defer func() { EndAttempts = was }() + for _, c := range []struct { + name string + data func() []byte + prior int + }{ + {"poison", func() []byte { raw, _ := sdkstep.EncodeStepAsk(testAsk(sdkstep.AskDelivery)); return raw }, sdkstep.AskRuns}, + {"unread with an id", func() []byte { return []byte(`{"id":"mwn1abcdef0123","kind":"check","v":99}`) }, 0}, + } { + t.Run(c.name, func(t *testing.T) { + src := NewFakeAsks() + src.Publish(sdkstep.AskCheck, c.data()) + if c.prior > 0 { // a delivery of the delivery queue + src = NewFakeAsks() + src.Publish(sdkstep.AskDelivery, c.data()) + } + var mu sync.Mutex + puts := 0 + put := func(string, any) (uint64, error) { + mu.Lock() + defer mu.Unlock() + puts++ + if puts > 1 || c.prior == 0 { // the start is written, the end is not + return 0, errors.New("the state is full") + } + return 1, nil + } + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Prior: func(string) int { return c.prior }, Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the ask given back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 }) + if acked, termed, _ := src.Fates(); len(acked)+len(termed) != 0 { + t.Errorf("acked %v termed %v, want none: no end was written", acked, termed) + } + }) + } +} + +// A-489a: a refused run (Declining) and a dropped ask each say their pipeline step's end as could not +// run, naming its pipeline, and are acknowledged once it is written. +func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) { + for _, c := range []string{"declined in its run", "dropped"} { + t.Run(c, func(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + a := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(), + Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepLint, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40), + Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}} + raw, eerr := sdkstep.EncodeStepAsk(a) + if eerr != nil { + t.Fatal(eerr) + } + src.Publish(sdkstep.AskCheck, raw) + var mu sync.Mutex + var ended []sdkstep.StepEnded + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Say: func(event string, body any, id string) error { + if event == "step-ended" { + var e sdkstep.StepEnded + b, _ := json.Marshal(body) + if json.Unmarshal(b, &e) == nil { + mu.Lock() + ended = append(ended, e) + mu.Unlock() + } + } + return nil + }, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { + return nil, &Declining{Text: "this node builds with another toolchain", Lacks: []string{"the toolchain"}} + }} + if c == "dropped" { + drop, _ := json.Marshal(nodebuildgo.Drop{V: int64(sdkstep.Version), By: "mesh-delivery-builder", Why: "superseded", At: ptr(time.Now())}) + p.Load = func(key string) ([]byte, error) { + if key == sdkstep.DropKey(a.ID) { + return drop, nil + } + return nil, nil + } + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) >= 1 }) + waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 }) + mu.Lock() + e := ended[0] + mu.Unlock() + if e.Outcome != sdkstep.OutcomeCouldNotRun || e.Pipeline != "main.12.m" { + t.Errorf("the end said %+v, want could not run naming its pipeline", e) + } + if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 { + t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked) + } + }) } } From 13db05ce601ffdb08a2c5784045d16b7b33958cc Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 15:45:48 +0100 Subject: [PATCH 6/7] steprun: the decline of a pipeline step is held by a test; RunsCounted cites A-489c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §24; lead answers A-489a, A-489c Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../steprun/conformance/conformance.go | 2 +- .../steprun/pull_poison_test.go | 21 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go index 2338afa1..e3f8ade3 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go @@ -291,7 +291,7 @@ func StepEnds(t *testing.T, steps steprun.AskSteps) { } } -// RunsCounted is the case every build module runs on its puller (A-489a): the poison rule and a transport fault's run count the +// A-489c: RunsCounted is the case every build module runs on its puller (A-489a): the poison rule and a transport fault's run count the // runs the module's steps state says started, so the puller is built with Prior over the module's own read of that state. The // module builds its puller with read as its steps state's read; a step key it holds with a started run must count as that run. func RunsCounted(t *testing.T, puller func(read func(key string) (json.RawMessage, error)) *steprun.Puller) { diff --git a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go index 974cb160..6ec823d4 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go @@ -136,10 +136,10 @@ func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) { } } -// A-489a: a refused run (Declining) and a dropped ask each say their pipeline step's end as could not +// A-489a: a decline at the take (a need the node lacks), a refused run (Declining) and a dropped ask each say their pipeline step's end as could not // run, naming its pipeline, and are acknowledged once it is written. func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) { - for _, c := range []string{"declined in its run", "dropped"} { + for _, c := range []string{"declined", "declined in its run", "dropped"} { t.Run(c, func(t *testing.T) { src, st := NewFakeAsks(), newSteps() a := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(), @@ -177,9 +177,20 @@ func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) { return nil, nil } } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go p.Pull(ctx) + if c == "declined" { + // The decoder refuses needs on a pipeline step ask, so no ask of the queue reaches the decline; the line is held + // by calling it with the ask built whole. + acked := 0 + p.decline(&Pulled{Ack: func() error { acked++; return nil }, Term: func() error { return nil }, Nak: func() error { return nil }}, a, []string{"node-build-node"}) + if acked != 1 { + t.Errorf("acked %d times, want once", acked) + } + src.Acked = append(src.Acked, "declined") + } else { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + } waitFor(t, "the end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) >= 1 }) waitFor(t, "the ask acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 1 }) mu.Lock() From 7617dda5f67e4bc8f94b66176d896a6c494a3f89 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 16:27:26 +0100 Subject: [PATCH 7/7] steprun: an ask naming no recipe is not declined for its recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner names a step ask's toolchain by language and version; every node sat out every TypeScript step. Builds: design 47 G §416 (A-104) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- modules/mesh-delivery-sdk/steprun/askrun.go | 6 +++-- .../mesh-delivery-sdk/steprun/askrun_test.go | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/askrun.go b/modules/mesh-delivery-sdk/steprun/askrun.go index 01525731..74577abd 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun.go +++ b/modules/mesh-delivery-sdk/steprun/askrun.go @@ -165,7 +165,9 @@ func (s AskSteps) fetch(ctx context.Context, fetch *sdkstep.Fetch) (json.RawMess // onToolchain is the toolchain this node builds with when the ask names one and this module reports its own, nil when // either is absent. A node on another toolchain than the ask names answers a Declining: it is not on the agreed one and -// sits out, for a node that is to take the ask (novox/hq ADR 0584 clause 2). +// sits out, for a node that is to take the ask (novox/hq ADR 0584 clause 2). The recipe is compared only when the ask names +// one: the runner names a step ask's language and version alone, the recipe and the node's digest being the seat's to report, +// not the runner's to know (design 47 G §416, A-104). func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed) (*sdkstep.ToolchainUsed, error) { if want == nil || s.Toolchain == nil { return nil, nil @@ -174,7 +176,7 @@ func (s AskSteps) onToolchain(ctx context.Context, want *sdkstep.ToolchainUsed) if err != nil { return nil, err } - if !sdkstep.SameToolchainVersion(tc.Version, want.Version) || tc.Recipe != want.Recipe { + if !sdkstep.SameToolchainVersion(tc.Version, want.Version) || (want.Recipe != "" && tc.Recipe != want.Recipe) { return nil, &Declining{Lacks: []string{fmt.Sprintf("the agreed toolchain %s (recipe %s)", sdkstep.LanguageVersion(want.Version), want.Recipe)}, Text: fmt.Sprintf("the ask is for %s (recipe %s) and this node builds with %s (recipe %s): it sits out, and the ask is left to a node on the agreed toolchain", sdkstep.LanguageVersion(want.Version), want.Recipe, sdkstep.LanguageVersion(tc.Version), tc.Recipe)} diff --git a/modules/mesh-delivery-sdk/steprun/askrun_test.go b/modules/mesh-delivery-sdk/steprun/askrun_test.go index 4fb03b81..f868984f 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun_test.go +++ b/modules/mesh-delivery-sdk/steprun/askrun_test.go @@ -512,3 +512,27 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { } } } + +// Design 47 G §416, A-104: the runner names a step ask's toolchain by language and version alone, so a node on that version +// takes it whatever its recipe; an ask that names a recipe still declines a node on another. +func TestAnAskNamingNoRecipeIsNotDeclinedForItsRecipe(t *testing.T) { + for _, c := range []struct { + name string + want sdkstep.ToolchainUsed + decline bool + }{ + {"version, no recipe", sdkstep.ToolchainUsed{Version: "go1.26"}, false}, + {"the same recipe", sdkstep.ToolchainUsed{Version: "go1.26", Recipe: "3"}, false}, + {"another recipe", sdkstep.ToolchainUsed{Version: "go1.26", Recipe: "4"}, true}, + {"another version, no recipe", sdkstep.ToolchainUsed{Version: "go1.27"}, true}, + } { + s := AskSteps{Toolchain: func(context.Context) (sdkstep.ToolchainUsed, error) { + return sdkstep.ToolchainUsed{Version: "go1.26.2", Recipe: "3"}, nil + }} + _, err := s.onToolchain(context.Background(), &c.want) + var dec *Declining + if errors.As(err, &dec) != c.decline || (err != nil && !c.decline) { + t.Errorf("%s: err %v, want declined %v", c.name, err, c.decline) + } + } +}