Cap the forge wait as a whole, and wait out only a forge not served (novox/hq issue 486)
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered

The registration holds the controller's one receive loop. Checking the bound
only between asks let an ask in flight hold it near two minutes, the
deaf-controller limit. The whole wait now runs under a 60 s deadline, still
says it waited when that deadline ends it, and a tool that did not answer in
time is not asked again: it has already held the loop its whole wait.
This commit is contained in:
2026-10-12 11:32:14 +02:00
parent 47205403d3
commit 4aacd98f8d
2 changed files with 77 additions and 36 deletions
+37 -21
View File
@@ -172,10 +172,10 @@ type waitOut struct {
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
// seconds, so 90 s waits one out with room to spare, and stays short because the registration holds the
// controller's receive loop while it waits. The pauses are those the build machine waits the registry with
// (novox/hq issue 457).
var forgeWait = waitOut{bound: 90 * time.Second, first: time.Second, most: 10 * time.Second,
// about 10 s, so 60 s waits one out with room to spare. It is a cap on the whole wait, an ask in flight included:
// the registration holds the controller's one receive loop while it waits, and a loop held two minutes reads as
// deaf. The pauses are those the build machine waits the registry with (novox/hq issue 457).
var forgeWait = waitOut{bound: 60 * time.Second, first: time.Second, most: 10 * time.Second,
now: time.Now, sleep: sleepOrDone}
// sleepOrDone pauses for d, or less when ctx ends first.
@@ -190,37 +190,53 @@ func sleepOrDone(ctx context.Context, d time.Duration) error {
}
}
// forgeNotNow is an answer that says the forge's tool is not there now, not that the forge refused: nothing
// serves it (the tool runner re-serving), or it did not answer within the wait. Anything else — a refusal, an
// unreadable answer, a repository with no id — is the forge's word, and is never asked again.
func forgeNotNow(err error) bool {
return errors.Is(err, link.ErrNothingServes) || errors.Is(err, link.ErrNoAnswer)
}
// forgeNotNow is an answer that says the forge's tool is not served now — the tool runner re-serving — not that
// the forge refused. Only that is asked again: a refusal, an unreadable answer or a repository with no id is the
// forge's word, and a tool that did not answer in time has already held the receive loop its whole wait.
func forgeNotNow(err error) bool { return errors.Is(err, link.ErrNothingServes) }
// forgeWaitedOut asks the forge about repository, asking again within w's bound while it answers "not now"
// (novox/hq issue 486). Past the bound it fails, saying it waited, for how long, and what the forge last said.
// forgeWaitedOut asks the forge about repository, asking again while it answers "not served now", all of it
// within w's bound, an ask in flight included (novox/hq issue 486). Past the bound it fails, saying it waited,
// for how long, and what the forge last said; when the caller's own context ends, it says that instead.
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
capped, cancel := context.WithTimeout(ctx, w.bound)
defer cancel()
began := w.now()
pause := w.first
for asked := 1; ; asked++ {
facts, err := ask(ctx)
if err == nil || !forgeNotNow(err) || ctx.Err() != nil {
if err == nil && asked > 1 {
facts, err := ask(capped)
if err == nil {
if asked > 1 {
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
w.now().Sub(began).Round(time.Second))
}
return facts, err
return facts, nil
}
waited := w.now().Sub(began)
if waited+pause > w.bound {
if ctx.Err() != nil {
return forgeFacts{}, err // the caller's own context ended: no wait of ours to speak of
}
gaveUp := func() (forgeFacts, error) {
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
"did not (novox/hq issue 486): %w", waited.Round(time.Second), repository, asked, err)
"did not (novox/hq issue 486): %w", w.now().Sub(began).Round(time.Second), repository, asked, err)
}
if capped.Err() != nil {
return gaveUp() // the bound ended an ask in flight
}
if !forgeNotNow(err) {
return forgeFacts{}, err
}
if w.now().Sub(began)+pause > w.bound {
return gaveUp()
}
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
if err := w.sleep(ctx, pause); err != nil {
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository, err)
if serr := w.sleep(capped, pause); serr != nil {
if ctx.Err() != nil {
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository,
ctx.Err())
}
return gaveUp()
}
pause = min(2*pause, w.most)
}