Archived
Cap the forge wait as a whole, and wait out only a forge not served (novox/hq issue 486)
The registration holds the controller's one receive loop. Checking the bound only between asks let an ask in flight hold it near two minutes, the deaf-controller limit. The whole wait now runs under a 60 s deadline, still says it waited when that deadline ends it, and a tool that did not answer in time is not asked again: it has already held the loop its whole wait.
This commit is contained in:
@@ -172,10 +172,10 @@ type waitOut struct {
|
||||
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
|
||||
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
|
||||
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
|
||||
// seconds, so 90 s waits one out with room to spare, and stays short because the registration holds the
|
||||
// controller's receive loop while it waits. The pauses are those the build machine waits the registry with
|
||||
// (novox/hq issue 457).
|
||||
var forgeWait = waitOut{bound: 90 * time.Second, first: time.Second, most: 10 * time.Second,
|
||||
// about 10 s, so 60 s waits one out with room to spare. It is a cap on the whole wait, an ask in flight included:
|
||||
// the registration holds the controller's one receive loop while it waits, and a loop held two minutes reads as
|
||||
// deaf. The pauses are those the build machine waits the registry with (novox/hq issue 457).
|
||||
var forgeWait = waitOut{bound: 60 * time.Second, first: time.Second, most: 10 * time.Second,
|
||||
now: time.Now, sleep: sleepOrDone}
|
||||
|
||||
// sleepOrDone pauses for d, or less when ctx ends first.
|
||||
@@ -190,37 +190,53 @@ func sleepOrDone(ctx context.Context, d time.Duration) error {
|
||||
}
|
||||
}
|
||||
|
||||
// forgeNotNow is an answer that says the forge's tool is not there now, not that the forge refused: nothing
|
||||
// serves it (the tool runner re-serving), or it did not answer within the wait. Anything else — a refusal, an
|
||||
// unreadable answer, a repository with no id — is the forge's word, and is never asked again.
|
||||
func forgeNotNow(err error) bool {
|
||||
return errors.Is(err, link.ErrNothingServes) || errors.Is(err, link.ErrNoAnswer)
|
||||
}
|
||||
// forgeNotNow is an answer that says the forge's tool is not served now — the tool runner re-serving — not that
|
||||
// the forge refused. Only that is asked again: a refusal, an unreadable answer or a repository with no id is the
|
||||
// forge's word, and a tool that did not answer in time has already held the receive loop its whole wait.
|
||||
func forgeNotNow(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
||||
|
||||
// forgeWaitedOut asks the forge about repository, asking again within w's bound while it answers "not now"
|
||||
// (novox/hq issue 486). Past the bound it fails, saying it waited, for how long, and what the forge last said.
|
||||
// forgeWaitedOut asks the forge about repository, asking again while it answers "not served now", all of it
|
||||
// within w's bound, an ask in flight included (novox/hq issue 486). Past the bound it fails, saying it waited,
|
||||
// for how long, and what the forge last said; when the caller's own context ends, it says that instead.
|
||||
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
|
||||
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
|
||||
capped, cancel := context.WithTimeout(ctx, w.bound)
|
||||
defer cancel()
|
||||
began := w.now()
|
||||
pause := w.first
|
||||
for asked := 1; ; asked++ {
|
||||
facts, err := ask(ctx)
|
||||
if err == nil || !forgeNotNow(err) || ctx.Err() != nil {
|
||||
if err == nil && asked > 1 {
|
||||
facts, err := ask(capped)
|
||||
if err == nil {
|
||||
if asked > 1 {
|
||||
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
|
||||
w.now().Sub(began).Round(time.Second))
|
||||
}
|
||||
return facts, err
|
||||
return facts, nil
|
||||
}
|
||||
waited := w.now().Sub(began)
|
||||
if waited+pause > w.bound {
|
||||
if ctx.Err() != nil {
|
||||
return forgeFacts{}, err // the caller's own context ended: no wait of ours to speak of
|
||||
}
|
||||
gaveUp := func() (forgeFacts, error) {
|
||||
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
|
||||
"did not (novox/hq issue 486): %w", waited.Round(time.Second), repository, asked, err)
|
||||
"did not (novox/hq issue 486): %w", w.now().Sub(began).Round(time.Second), repository, asked, err)
|
||||
}
|
||||
if capped.Err() != nil {
|
||||
return gaveUp() // the bound ended an ask in flight
|
||||
}
|
||||
if !forgeNotNow(err) {
|
||||
return forgeFacts{}, err
|
||||
}
|
||||
if w.now().Sub(began)+pause > w.bound {
|
||||
return gaveUp()
|
||||
}
|
||||
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
|
||||
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
|
||||
if err := w.sleep(ctx, pause); err != nil {
|
||||
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository, err)
|
||||
if serr := w.sleep(capped, pause); serr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository,
|
||||
ctx.Err())
|
||||
}
|
||||
return gaveUp()
|
||||
}
|
||||
pause = min(2*pause, w.most)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user