diff --git a/modules/mesh-delivery-sdk/steprun/askrun.go b/modules/mesh-delivery-sdk/steprun/askrun.go index 93f18ec5..01525731 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun.go +++ b/modules/mesh-delivery-sdk/steprun/askrun.go @@ -57,7 +57,7 @@ type AskSteps struct { SayStep func(end sdkstep.StepEnded) error } -// SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to +// A-489b (G-22-lead-answers-2.md): SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to // publish it (design 47 G *Asked on a work queue, ended by an event*, §24). A build module whose steps cannot is refused where // it is built (conformance.StepEnds, which every build module's test runs, and the catalogue check failurekit), never // discovered live as a step ask refused with "says no step's end". diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go index 05d3a4f1..a8eb5b33 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go @@ -279,7 +279,7 @@ func judgePassing(c PassingCase, failed Failed) string { return "" } -// StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end +// A-489b: StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end // (steprun.AskSteps.SaysItsEnds), or the module would refuse every step ask the runner sends, as mesh-build-node did on its // first live pipeline (design 47 G §24). A catalogue check (failurekit) refuses a build module whose tests do not call it. func StepEnds(t *testing.T, steps steprun.AskSteps) { diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 2bc1b2ed..15418a1a 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -32,6 +32,8 @@ type Pulled struct { Subject string // Delivered is how many times the bus has handed it over, this time included. Delivered uint64 + // Sequence is its sequence in the stream (0 for a source that has none), which the journal names for an ask that cannot be read. + Sequence uint64 // Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its // acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it. Ack, InProgress, Nak, Term func() error @@ -580,7 +582,8 @@ func (p *Puller) decline(msg *Pulled, ask sdkstep.StepAsk, lacks []string) { return } p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", ")) - _ = msg.Term() + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "this node lacks "+strings.Join(lacks, ", ")) + _ = msg.Ack() // A-489a: its end is written p.mu.Lock() if p.declined == nil { p.declined = map[string]bool{} @@ -602,7 +605,7 @@ func (p *Puller) declineRun(msg *Pulled, ask sdkstep.StepAsk, key string, step s return } p.logf("the ask %s is declined on %s: %s", ask.ID, p.Node, d.Text) - _ = msg.Term() + _ = msg.Ack() // A-489a: its end is written (run() says it) p.mu.Lock() if p.declined == nil { p.declined = map[string]bool{} @@ -637,8 +640,8 @@ func (p *Puller) refuseUnread(msg *Pulled, why error) { } } } - p.logf("an ask dropped: %v", why) - _ = msg.Term() + p.logf("an ask dropped, no id to answer it by, acknowledged with this line as its only record (stream sequence %d): %v", msg.Sequence, why) + _ = msg.Ack() // A-489a } func withKind(msg *Pulled, kind string) *Pulled { @@ -648,6 +651,37 @@ func withKind(msg *Pulled, kind string) *Pulled { return msg } +// priorRuns is how many runs of the ask started before this delivery: its step key's (Puller.Prior), else the bus's delivery +// count less this one. The bus's count also counts an ask given back before it ran, which is no run. +func (p *Puller) priorRuns(msg *Pulled, ask sdkstep.StepAsk) int { + if p.Prior != nil { + if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil { + return p.Prior(akey) + } + } + return int(msg.Delivered) - 1 +} + +// endUnrun ends an ask that is run no more (A-489, A-489a): poison when runs of it started and none ended, out of its +// deliveries when none did. Its step ends and its step-ended is said, and only then it is acknowledged; an end that could not +// be written gives it back, never acknowledged. +func (p *Puller) endUnrun(msg *Pulled, ask sdkstep.StepAsk, key string, step sdkstep.Step, prior int) { + var why error + if prior > 0 { + why = fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior) + } else { + why = fmt.Errorf("the ask is on its last delivery (%d) and no run of it started: it is out of its deliveries and run no more", msg.Delivered) + } + p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why) + if _, ferr := finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf); ferr != nil { + p.logf("the end of the ask %s could not be written on %s, so it is given back: %v", ask.ID, p.Node, ferr) + _ = msg.Nak() + return + } + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error()) + _ = msg.Ack() +} + // take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id. func (p *Puller) take(ctx context.Context, msg *Pulled) { kind := msg.Kind @@ -662,6 +696,18 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.refuseUnread(msg, err) return } + if msg.Delivered >= sdkstep.AskMaxDeliver { + // Its last delivery (A-489a): the bus gives it no more, so it is ended here, run or not, never given back. + prior := p.priorRuns(msg, ask) + step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1) + if err != nil { + p.logf("the ask %s is on its last delivery and cannot start its end on %s: %v", ask.ID, p.Node, err) + _ = msg.Nak() + return + } + p.endUnrun(msg, ask, key, step, prior) + return + } if why, judged := p.askerRefusal(msg, ask); !judged { p.giveBack(msg, ask) return @@ -677,14 +723,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.decline(msg, ask, lacks) return } - // The run this is: the runs of the ask that started before it, from its step key, and one more. The bus's delivery count - // also counts an ask given back before it ran (a seat list not read yet, a start that failed), which is no run. - prior := int(msg.Delivered) - 1 - if p.Prior != nil { - if akey, kerr := sdkstep.StepKeyOf(keyAsker(msg), p.Node, ask.ID); kerr == nil { - prior = p.Prior(akey) - } - } + prior := p.priorRuns(msg, ask) step, key, err := beginRun(p.Put, keyAsker(msg), p.Node, ask.Kind+" ask", ask.ID, buildrun.AskWhat(askRead(ask)), prior+1) if err != nil { p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err) @@ -692,13 +731,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { return } if prior >= sdkstep.AskRuns { - // Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said - // (issue 1907): the asker's key fails with why. - why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", prior) - p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why) - _, _ = finish(p.Put, p.KeepOutput, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then - p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, why.Error()) // and its asker, a pipeline's runner too, hears it - _ = msg.Ack() // A-489 + p.endUnrun(msg, ask, key, step, prior) return } module := "" @@ -781,7 +814,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key var dec *Declining if errors.As(err, &dec) && ctx.Err() == nil { p.declineRun(msg, ask, key, step, dec) - p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text, nil, "") + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text) return } if ctx.Err() != nil { diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index 5669ec4b..08e9a1b8 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -116,11 +116,11 @@ func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait ti if !ok { return nil, batch.Error() } - var delivered uint64 = 1 + var delivered, sequence uint64 = 1, 0 if meta, err := msg.Metadata(); err == nil && meta != nil { - delivered = meta.NumDelivered + delivered, sequence = meta.NumDelivered, meta.Sequence.Stream } - return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, + return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, Sequence: sequence, Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term, NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil case <-ctx.Done(): diff --git a/modules/mesh-delivery-sdk/steprun/pull_drop.go b/modules/mesh-delivery-sdk/steprun/pull_drop.go index 96d82825..af837dbd 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_drop.go +++ b/modules/mesh-delivery-sdk/steprun/pull_drop.go @@ -147,7 +147,8 @@ func (p *Puller) refuseDropped(msg *Pulled, ask sdkstep.StepAsk, d *nodebuildgo. return } p.logf("the ask %s is refused on %s: it was dropped (%s)", ask.ID, p.Node, d.Why) - _ = msg.Term() + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "it was dropped: "+d.Why) + _ = msg.Ack() // A-489a: its end is written p.forgetDrop(ask.ID) } diff --git a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go index b39697fc..bfa165f8 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go @@ -3,6 +3,7 @@ package steprun import ( "context" "encoding/json" + "errors" "strings" "sync" "testing" @@ -24,11 +25,13 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { name string delivered uint64 prior int - poisoned bool + ends string // what its end says; "" when it must run }{ - {"given back four times, never run", sdkstep.AskRuns + 2, 0, false}, - {"three runs started and none ended", 3, sdkstep.AskRuns, true}, - {"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, true}, + {"given back three times, never run", sdkstep.AskRuns + 1, 0, ""}, + {"three runs started and none ended", 3, sdkstep.AskRuns, "poison"}, + {"three runs started, delivered many times", sdkstep.AskRuns + 4, sdkstep.AskRuns, "poison"}, + {"its last delivery, given back before and never run", sdkstep.AskMaxDeliver, 0, "out of its deliveries"}, + {"its last delivery after a run that started", sdkstep.AskMaxDeliver, 1, "poison"}, } { t.Run(c.name, func(t *testing.T) { src, st := NewFakeAsks(), newSteps() @@ -60,17 +63,20 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - if c.poisoned { + if c.ends != "" { waitFor(t, "the poisoned ask's end said", func() bool { mu.Lock(); defer mu.Unlock(); return len(ended) == 1 }) mu.Lock() e := ended[0] mu.Unlock() - if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, "poison") || e.Pipeline != "main.12.m" { + if e.Outcome != sdkstep.OutcomeCouldNotRun || !strings.Contains(e.Said, c.ends) || e.Pipeline != "main.12.m" { t.Fatalf("the end said %+v, want could not run, poison, naming its pipeline", e) } + if acked, termed, naked := src.Fates(); len(acked) != 1 || len(termed)+len(naked) != 0 { + t.Errorf("acked %v termed %v naked %v, want it acknowledged once its end is written (A-489a)", acked, termed, naked) + } select { case <-ran: - t.Fatal("a poisoned ask ran") + t.Fatal("an ended ask ran") default: } return @@ -83,3 +89,71 @@ func TestOnlyRunsThatStartedCountTowardPoison(t *testing.T) { }) } } + +// A-489a: an end that could not be written is given back, never acknowledged. +func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) { + src := NewFakeAsks() + src.Delivered = sdkstep.AskMaxDeliver + _ = src.PublishAsk(testAsk(sdkstep.AskDelivery)) + put := func(string, any) (uint64, error) { return 0, errors.New("the state is full") } + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the ask given back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 }) + if acked, _, _ := src.Fates(); len(acked) != 0 { + t.Errorf("acked %v, want none: no end was written", acked) + } +} + +// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the +// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after 4 minutes), not 3. +func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + src.Delivered = 3 + a := testAsk(sdkstep.AskDelivery) + _ = src.PublishAsk(a) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Prior: func(string) int { return 1 }, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { + return nil, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")} + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the ask given again", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) == 1 }) + key, _ := sdkstep.AskStepKey("", "shanks", a.ID) + s, _ := st.get(key) + src.mu.Lock() + delays := append([]time.Duration(nil), src.Delays...) + src.mu.Unlock() + if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != sdkstep.AgainAfter(2) { + t.Errorf("step %+v, delays %v, want again in run 2 after %s", s, delays, sdkstep.AgainAfter(2)) + } +} + +// Puller.Prior over a steps state's read: the Run of the step the key holds, 0 for an absent key, a failed read or what is no +// step, and nil for no read. +func TestPriorRunsReadsTheRunOfTheStepAKeyHolds(t *testing.T) { + read := func(key string) (json.RawMessage, error) { + switch key { + case "running": + return json.RawMessage(`{"id":"x","run":2}`), nil + case "junk": + return json.RawMessage(`[`), nil + case "broken": + return nil, errors.New("no bus") + } + return nil, nil + } + prior := PriorRuns(read) + for key, want := range map[string]int{"running": 2, "absent": 0, "junk": 0, "broken": 0} { + if got := prior(key); got != want { + t.Errorf("PriorRuns(%s) = %d, want %d", key, got, want) + } + } + if PriorRuns(nil) != nil { + t.Error("PriorRuns(nil) is not nil") + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_test.go b/modules/mesh-delivery-sdk/steprun/pull_test.go index d5dd4d0d..f5cc9578 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_test.go @@ -244,7 +244,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "three ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 3 }) + waitFor(t, "three acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 3 }) if ran { t.Fatal("a wrong or a declined ask ran") } @@ -269,7 +269,7 @@ func TestPullDropsWhatIsNoAsk(t *testing.T) { return nil, &Declining{Text: "the ask is for 26.13.0 and this node builds with 26.12.1", Lacks: []string{"the agreed toolchain 26.13.0 (recipe 3)"}} }} go s2.Pull(ctx) - waitFor(t, "the ask termed on the node that sits out", func() bool { _, tm, _ := sit.Fates(); return len(tm) == 1 }) + waitFor(t, "the ask acknowledged on the node that sits out", func() bool { ak, _, _ := sit.Fates(); return len(ak) == 1 }) sk, _ := sdkstep.AskStepKey("", "ace", toolchain.ID) ss, _ := sitSteps.get(sk) if d, ok := sdkstep.DeclinedOf(ss); !ok || len(d.Lacks) != 1 || !strings.Contains(ss.Reason, "26.12.1") { @@ -620,7 +620,7 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) - waitFor(t, "both ended", func() bool { ak, tm, _ := src.Fates(); return len(ak)+len(tm) == 2 }) + waitFor(t, "both acknowledged", func() bool { ak, _, _ := src.Fates(); return len(ak) == 2 }) key, _ := sdkstep.AskStepKey("", "ace", good.ID) s, ok := st.get(key) if !ok || s.State != sdkstep.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") {