From 374d060cc0be8a07fd2bc127260d79c61d8f727b Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:57:44 +0100 Subject: [PATCH] Take an ask's bound that expired while the runner was down (H-A-238) The bounds were watched for updates only, so an expiry marker written while the runner was down was never heard and that ask had no bound. At the start each ask key's last entry is read; an expiry marker is taken then. A running heard before the seat-holders view is read is heard again after a delay instead of at once. Builds: design 47 H H-A-238 (5), H-A-243 (1) --- .../cmd/mesh-delivery-pipeline/askbounds.go | 5 ++- .../cmd/mesh-delivery-pipeline/main.go | 2 +- .../queue_integration_test.go | 23 +++++++++++ .../cmd/mesh-delivery-pipeline/steps.go | 38 +++++++++++++++++-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go index 94d7323e..a6accd91 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -112,6 +112,9 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ... }) } +// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again. +const holdersNotReadDelay = 2 * time.Second + // hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)). func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error { return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { @@ -128,7 +131,7 @@ func hearHolderStarted(hub handlers, started func(module string) (int, error), l n, err := started(ev.Module) switch { case errors.Is(err, pipeline.ErrHoldersNotRead): - _ = m.Nak() // heard again once the view is read + _ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then return case err != nil: logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index f5f8af67..8148e161 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dropExpired(key) } } - if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { + if err := watchBounds(ctx, bounds, expired, missedAskBound(ctx, bounds), logger.Printf); err != nil { return err } if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 84a5e92a..e9c797a1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -403,7 +403,30 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if build.Ask == "" || build.Started == nil { t.Fatalf("main.10's build = %+v, want it asked", build) } + // Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is + // read and taken at once, never a fresh bound's full length later. + boundsKV, err := js.KeyValue(ctx, module+"_bounds") + if err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return err == nil + }) + stopRunner() + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return errors.Is(err, jetstream.ErrKeyNotFound) + }) + restarted := time.Now() + stopRunner = start() outcome("main.10", seat.PipelineOutcomeFailed) + if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second { + t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted) + } + if !strings.Contains(logs.String(), "expired while the runner was away: taken now") { + t.Fatal("the start did not say the bound it took") + } s := stepFor(get("main.10"), "app", seat.StepResultStepBuild) if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second { t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index 81b53a6a..4d5b85e7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -182,13 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline. } // watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235, -// H-A-238 (5)): no timer in the process. -func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { - w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) +// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the +// watch began: the watch's first entries, each key's last, are read to their end, and each ask's key whose last entry is +// an expiry marker is taken as expired now (missed tells it); a delete marker is a first word, never an expiry. Then every +// expiry is heard as it comes. +func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), missed func(key string) bool, logf func(string, ...any)) error { + w, err := bounds.WatchAll(ctx) if err != nil { return err } go func() { + initial := true for { select { case <-ctx.Done(): @@ -198,9 +202,19 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke if !ok { return } - if e == nil || e.Operation() != jetstream.KeyValuePurge { + if e == nil { + initial = false // the last entries are read: what follows is new continue } + if e.Operation() != jetstream.KeyValuePurge { + continue + } + if initial { + if _, _, _, isAsk := pipeline.ParseAskBound(e.Key()); !isAsk || !missed(e.Key()) { + continue + } + logf("the bound %s expired while the runner was away: taken now", e.Key()) + } expired(e.Key()) } } @@ -208,6 +222,22 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke return nil } +// missedAskBound says whether an ask's bound found expired at the start is still to be taken: a first bound whose ask was +// asked again already (its second bound is held) is not, so a start never asks it once more again. +func missedAskBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) bool { + return func(key string) bool { + onSeat, ask, again, ok := pipeline.ParseAskBound(key) + if !ok { + return false + } + if again { + return true + } + e, err := bounds.Get(ctx, pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))) + return err != nil || e.Operation() != jetstream.KeyValuePut + } +} + // keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a // name or lies beneath it, in one pass over the bucket. func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) {