Wait out a forge not served for a moment when registering a build (novox/hq issue 486)
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request

A registration asked the forge while node-tools re-served after a tool-runner
rollout, heard "nothing serves gitea.gitea_api", and failed a good build's
plan. The bus's not-served answer is now link.ErrNothingServes (and a timeout
link.ErrNoAnswer), and the registration asks again for up to 90 s on those
alone, never on a refusal; past the bound it fails saying how long it waited.
This commit is contained in:
2026-10-12 11:23:03 +02:00
parent da1473e9db
commit 47205403d3
5 changed files with 227 additions and 7 deletions
+74 -1
View File
@@ -153,8 +153,81 @@ type forgeFacts struct {
}
// askTheForge asks the forge, through its module's tools on the bus, for a repository's id and its branch's
// protection. A variable so a test needs no forge.
// protection, waiting out a forge not served for a moment (novox/hq issue 486). A variable so a test needs no
// forge.
var askTheForge = func(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
return forgeWaitedOut(ctx, forgeWait, owner+"/"+repo, func(ctx context.Context) (forgeFacts, error) {
return askTheForgeOnce(ctx, owner, repo, branch)
})
}
// waitOut bounds how long a question is asked again while its answer is "not now": a pause of first, doubling
// to most, for at most bound in all. now and sleep are the clock, so a test waits in no real time.
type waitOut struct {
bound, first, most time.Duration
now func() time.Time
sleep func(context.Context, time.Duration) error
}
// forgeWait is the registration's wait for the forge (novox/hq issue 486). On 2026-10-11 a build's registration
// asked the forge while node-tools on the control-node re-served after a rollout of the tool runner, heard
// "nothing serves gitea.gitea_api", and failed its plan; the forge answered again seconds later. A re-serve takes
// seconds, so 90 s waits one out with room to spare, and stays short because the registration holds the
// controller's receive loop while it waits. The pauses are those the build machine waits the registry with
// (novox/hq issue 457).
var forgeWait = waitOut{bound: 90 * time.Second, first: time.Second, most: 10 * time.Second,
now: time.Now, sleep: sleepOrDone}
// sleepOrDone pauses for d, or less when ctx ends first.
func sleepOrDone(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
return nil
}
}
// forgeNotNow is an answer that says the forge's tool is not there now, not that the forge refused: nothing
// serves it (the tool runner re-serving), or it did not answer within the wait. Anything else — a refusal, an
// unreadable answer, a repository with no id — is the forge's word, and is never asked again.
func forgeNotNow(err error) bool {
return errors.Is(err, link.ErrNothingServes) || errors.Is(err, link.ErrNoAnswer)
}
// forgeWaitedOut asks the forge about repository, asking again within w's bound while it answers "not now"
// (novox/hq issue 486). Past the bound it fails, saying it waited, for how long, and what the forge last said.
func forgeWaitedOut(ctx context.Context, w waitOut, repository string,
ask func(context.Context) (forgeFacts, error)) (forgeFacts, error) {
began := w.now()
pause := w.first
for asked := 1; ; asked++ {
facts, err := ask(ctx)
if err == nil || !forgeNotNow(err) || ctx.Err() != nil {
if err == nil && asked > 1 {
fmt.Printf("the forge answered about %s again, after %s (novox/hq issue 486)\n", repository,
w.now().Sub(began).Round(time.Second))
}
return facts, err
}
waited := w.now().Sub(began)
if waited+pause > w.bound {
return forgeFacts{}, fmt.Errorf("waited %s for the forge to answer about %s, asking %d times, and it "+
"did not (novox/hq issue 486): %w", waited.Round(time.Second), repository, asked, err)
}
fmt.Printf("the forge cannot be asked about %s now (%v); asking again in %s, for at most %s in all "+
"(novox/hq issue 486)\n", repository, err, pause, w.bound)
if err := w.sleep(ctx, pause); err != nil {
return forgeFacts{}, fmt.Errorf("stopped waiting for the forge to answer about %s: %w", repository, err)
}
pause = min(2*pause, w.most)
}
}
// askTheForgeOnce asks the forge once, through its module's tools on the bus.
func askTheForgeOnce(ctx context.Context, owner, repo, branch string) (forgeFacts, error) {
js, err := aBus()
if err != nil {
return forgeFacts{}, err