Take an ask's bound that expired while the runner was down (H-A-238)

The bounds were watched for updates only, so an expiry marker written
while the runner was down was never heard and that ask had no bound.
At the start each ask key's last entry is read; an expiry marker is
taken then. A running heard before the seat-holders view is read is
heard again after a delay instead of at once.

Builds: design 47 H H-A-238 (5), H-A-243 (1)
This commit is contained in:
2026-11-21 12:02:02 +01:00
parent 1f878eec58
commit 374d060cc0
4 changed files with 62 additions and 6 deletions
@@ -112,6 +112,9 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...
})
}
// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again.
const holdersNotReadDelay = 2 * time.Second
// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)).
func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error {
return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) {
@@ -128,7 +131,7 @@ func hearHolderStarted(hub handlers, started func(module string) (int, error), l
n, err := started(ev.Module)
switch {
case errors.Is(err, pipeline.ErrHoldersNotRead):
_ = m.Nak() // heard again once the view is read
_ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then
return
case err != nil:
logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err)
@@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
dropExpired(key)
}
}
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
if err := watchBounds(ctx, bounds, expired, missedAskBound(ctx, bounds), logger.Printf); err != nil {
return err
}
if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil {
@@ -403,7 +403,30 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
if build.Ask == "" || build.Started == nil {
t.Fatalf("main.10's build = %+v, want it asked", build)
}
// Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is
// read and taken at once, never a fresh bound's full length later.
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
if err != nil {
t.Fatal(err)
}
waitFor(t, ctx, logs, func() bool {
_, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask))
return err == nil
})
stopRunner()
waitFor(t, ctx, logs, func() bool {
_, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask))
return errors.Is(err, jetstream.ErrKeyNotFound)
})
restarted := time.Now()
stopRunner = start()
outcome("main.10", seat.PipelineOutcomeFailed)
if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second {
t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted)
}
if !strings.Contains(logs.String(), "expired while the runner was away: taken now") {
t.Fatal("the start did not say the bound it took")
}
s := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second {
t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s)
@@ -182,13 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline.
}
// watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235,
// H-A-238 (5)): no timer in the process.
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the
// watch began: the watch's first entries, each key's last, are read to their end, and each ask's key whose last entry is
// an expiry marker is taken as expired now (missed tells it); a delete marker is a first word, never an expiry. Then every
// expiry is heard as it comes.
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), missed func(key string) bool, logf func(string, ...any)) error {
w, err := bounds.WatchAll(ctx)
if err != nil {
return err
}
go func() {
initial := true
for {
select {
case <-ctx.Done():
@@ -198,9 +202,19 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePurge {
if e == nil {
initial = false // the last entries are read: what follows is new
continue
}
if e.Operation() != jetstream.KeyValuePurge {
continue
}
if initial {
if _, _, _, isAsk := pipeline.ParseAskBound(e.Key()); !isAsk || !missed(e.Key()) {
continue
}
logf("the bound %s expired while the runner was away: taken now", e.Key())
}
expired(e.Key())
}
}
@@ -208,6 +222,22 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke
return nil
}
// missedAskBound says whether an ask's bound found expired at the start is still to be taken: a first bound whose ask was
// asked again already (its second bound is held) is not, so a start never asks it once more again.
func missedAskBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) bool {
return func(key string) bool {
onSeat, ask, again, ok := pipeline.ParseAskBound(key)
if !ok {
return false
}
if again {
return true
}
e, err := bounds.Get(ctx, pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask)))
return err != nil || e.Operation() != jetstream.KeyValuePut
}
}
// keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a
// name or lies beneath it, in one pass over the bucket.
func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) {