jschoubben
acaf8830ec
A build or check failed now and then on 'Could not resolve host: novox.internal' between clones that resolved the same name, and its whole delivery failed with it. A clone whose lookup failed reached nothing, so it is cloned again with a growing pause for up to 60 s, each retry said in the build's log as the resolver not answering; past the bound it fails saying so. Any other git failure fails at once, as before. Covers a module's own clone, an artifact's context and a check's repositories.
115 lines
4.1 KiB
Go
115 lines
4.1 KiB
Go
package builder
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// A clone from the forge waits out a resolver that did not answer, for a bounded time (novox/hq issue 1773).
|
|
//
|
|
// **Why.** A build or a check failed now and then on "Could not resolve host: novox.internal" while cloning, between
|
|
// clones that resolved the same name a second before and after, and the whole delivery failed with it. The lookup
|
|
// failing is the resolver's passing fault, not the repository's: nothing reached the forge, so cloning again cannot
|
|
// do anything twice. This is issue 457's shape (a build waits out a registry held still) applied to the name lookup.
|
|
//
|
|
// **Only the lookup.** Any other failure of git — authentication, a repository or a ref that is not there, a 404 —
|
|
// fails at once, as it always did: trying it again would only make a real failure slower.
|
|
//
|
|
// **Bounded, and loud.** Each retry is said in the build's log as the resolver's failure, and past lookupWait the
|
|
// build fails saying the resolver did not answer for that long.
|
|
|
|
var (
|
|
// lookupWait is how long a clone waits for a resolver that does not answer.
|
|
lookupWait = 60 * time.Second
|
|
// lookupFirstPause is the first pause between tries; each pause doubles, up to lookupMostPause.
|
|
lookupFirstPause = time.Second
|
|
)
|
|
|
|
// lookupMostPause is the longest pause between two tries.
|
|
const lookupMostPause = 10 * time.Second
|
|
|
|
// lookupFailures are what git (through curl) and Go's resolver say when a name did not resolve, in lower case.
|
|
var lookupFailures = []string{
|
|
"could not resolve host",
|
|
"temporary failure in name resolution",
|
|
}
|
|
|
|
// failedLookup is whether an error is a name lookup that failed: git's "Could not resolve host", the system's
|
|
// "Temporary failure in name resolution", or a lookup that timed out.
|
|
func failedLookup(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
said := strings.ToLower(err.Error())
|
|
for _, f := range lookupFailures {
|
|
if strings.Contains(said, f) {
|
|
return true
|
|
}
|
|
}
|
|
return strings.Contains(said, "lookup ") && strings.Contains(said, "i/o timeout")
|
|
}
|
|
|
|
// cloneFromForge runs `git clone` of url into dest (git's arguments in args, run from dir), and while it fails on the
|
|
// name lookup, clones again with a growing pause until lookupWait has passed. What a failed try left at dest is
|
|
// removed before the next. Any other failure is returned at once.
|
|
func cloneFromForge(ctx context.Context, run Runner, dir, url, dest string, args []string,
|
|
say func(step, format string, args ...any)) error {
|
|
_, err := run(ctx, dir, "git", args...)
|
|
if !failedLookup(err) {
|
|
return err
|
|
}
|
|
started := time.Now()
|
|
pause := lookupFirstPause
|
|
for {
|
|
elapsed := time.Since(started)
|
|
left := lookupWait - elapsed
|
|
if left <= 0 {
|
|
say("clone", "the resolver did not answer for %s, cloning %s, for %s; the clone fails",
|
|
urlHost(url), url, elapsed.Round(time.Second))
|
|
return fmt.Errorf("the resolver did not answer a lookup of %s for %s, cloning %s: %w",
|
|
urlHost(url), elapsed.Round(time.Millisecond), url, err)
|
|
}
|
|
wait := min(pause, left)
|
|
say("clone", "the resolver did not answer for %s, cloning %s; trying again in %s, for up to %s "+
|
|
"(novox/hq issue 1773)", urlHost(url), url, wait, lookupWait)
|
|
select {
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("stopped while waiting for the resolver to answer for %s: %w (last: %v)",
|
|
urlHost(url), ctx.Err(), err)
|
|
case <-time.After(wait):
|
|
}
|
|
pause = min(pause*2, lookupMostPause)
|
|
if err := os.RemoveAll(dest); err != nil {
|
|
return err
|
|
}
|
|
if _, err = run(ctx, dir, "git", args...); !failedLookup(err) {
|
|
if err == nil {
|
|
say("clone", "the resolver answered for %s after %s; the clone worked", urlHost(url),
|
|
time.Since(started).Round(time.Millisecond))
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
// urlHost is the host (and port) a URL names, or the URL itself when it names none.
|
|
func urlHost(url string) string {
|
|
rest := url
|
|
if _, after, ok := strings.Cut(url, "://"); ok {
|
|
rest = after
|
|
}
|
|
if i := strings.IndexAny(rest, "/?#"); i >= 0 {
|
|
rest = rest[:i]
|
|
}
|
|
if i := strings.LastIndex(rest, "@"); i >= 0 {
|
|
rest = rest[i+1:]
|
|
}
|
|
if rest == "" {
|
|
return url
|
|
}
|
|
return rest
|
|
}
|