diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go index 94d7323e..a6accd91 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -112,6 +112,9 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ... }) } +// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again. +const holdersNotReadDelay = 2 * time.Second + // hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)). func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error { return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { @@ -128,7 +131,7 @@ func hearHolderStarted(hub handlers, started func(module string) (int, error), l n, err := started(ev.Module) switch { case errors.Is(err, pipeline.ErrHoldersNotRead): - _ = m.Nak() // heard again once the view is read + _ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then return case err != nil: logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index f5f8af67..8148e161 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dropExpired(key) } } - if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { + if err := watchBounds(ctx, bounds, expired, missedAskBound(ctx, bounds), logger.Printf); err != nil { return err } if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 84a5e92a..e9c797a1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -403,7 +403,30 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if build.Ask == "" || build.Started == nil { t.Fatalf("main.10's build = %+v, want it asked", build) } + // Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is + // read and taken at once, never a fresh bound's full length later. + boundsKV, err := js.KeyValue(ctx, module+"_bounds") + if err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return err == nil + }) + stopRunner() + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return errors.Is(err, jetstream.ErrKeyNotFound) + }) + restarted := time.Now() + stopRunner = start() outcome("main.10", seat.PipelineOutcomeFailed) + if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second { + t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted) + } + if !strings.Contains(logs.String(), "expired while the runner was away: taken now") { + t.Fatal("the start did not say the bound it took") + } s := stepFor(get("main.10"), "app", seat.StepResultStepBuild) if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second { t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index 81b53a6a..4d5b85e7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -182,13 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline. } // watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235, -// H-A-238 (5)): no timer in the process. -func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { - w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) +// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the +// watch began: the watch's first entries, each key's last, are read to their end, and each ask's key whose last entry is +// an expiry marker is taken as expired now (missed tells it); a delete marker is a first word, never an expiry. Then every +// expiry is heard as it comes. +func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), missed func(key string) bool, logf func(string, ...any)) error { + w, err := bounds.WatchAll(ctx) if err != nil { return err } go func() { + initial := true for { select { case <-ctx.Done(): @@ -198,9 +202,19 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke if !ok { return } - if e == nil || e.Operation() != jetstream.KeyValuePurge { + if e == nil { + initial = false // the last entries are read: what follows is new continue } + if e.Operation() != jetstream.KeyValuePurge { + continue + } + if initial { + if _, _, _, isAsk := pipeline.ParseAskBound(e.Key()); !isAsk || !missed(e.Key()) { + continue + } + logf("the bound %s expired while the runner was away: taken now", e.Key()) + } expired(e.Key()) } } @@ -208,6 +222,22 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke return nil } +// missedAskBound says whether an ask's bound found expired at the start is still to be taken: a first bound whose ask was +// asked again already (its second bound is held) is not, so a start never asks it once more again. +func missedAskBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) bool { + return func(key string) bool { + onSeat, ask, again, ok := pipeline.ParseAskBound(key) + if !ok { + return false + } + if again { + return true + } + e, err := bounds.Get(ctx, pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))) + return err != nil || e.Operation() != jetstream.KeyValuePut + } +} + // keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a // name or lies beneath it, in one pass over the bucket. func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) {