Try a forge clone again when its name lookup fails (issue 1773)

A build or check failed now and then on 'Could not resolve host:
novox.internal' between clones that resolved the same name, and its
whole delivery failed with it. A clone whose lookup failed reached
nothing, so it is cloned again with a growing pause for up to 60 s,
each retry said in the build's log as the resolver not answering;
past the bound it fails saying so. Any other git failure fails at
once, as before. Covers a module's own clone, an artifact's context
and a check's repositories.

internal/builder/clone_lookup.go

@@ -0,0 +1,114 @@
+package builder
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// A clone from the forge waits out a resolver that did not answer, for a bounded time (novox/hq issue 1773).
+//
+// **Why.** A build or a check failed now and then on "Could not resolve host: novox.internal" while cloning, between
+// clones that resolved the same name a second before and after, and the whole delivery failed with it. The lookup
+// failing is the resolver's passing fault, not the repository's: nothing reached the forge, so cloning again cannot
+// do anything twice. This is issue 457's shape (a build waits out a registry held still) applied to the name lookup.
+//
+// **Only the lookup.** Any other failure of git — authentication, a repository or a ref that is not there, a 404 —
+// fails at once, as it always did: trying it again would only make a real failure slower.
+//
+// **Bounded, and loud.** Each retry is said in the build's log as the resolver's failure, and past lookupWait the
+// build fails saying the resolver did not answer for that long.
+
+var (
+	// lookupWait is how long a clone waits for a resolver that does not answer.
+	lookupWait = 60 * time.Second
+	// lookupFirstPause is the first pause between tries; each pause doubles, up to lookupMostPause.
+	lookupFirstPause = time.Second
+)
+
+// lookupMostPause is the longest pause between two tries.
+const lookupMostPause = 10 * time.Second
+
+// lookupFailures are what git (through curl) and Go's resolver say when a name did not resolve, in lower case.
+var lookupFailures = []string{
+	"could not resolve host",
+	"temporary failure in name resolution",
+}
+
+// failedLookup is whether an error is a name lookup that failed: git's "Could not resolve host", the system's
+// "Temporary failure in name resolution", or a lookup that timed out.
+func failedLookup(err error) bool {
+	if err == nil {
+		return false
+	}
+	said := strings.ToLower(err.Error())
+	for _, f := range lookupFailures {
+		if strings.Contains(said, f) {
+			return true
+		}
+	}
+	return strings.Contains(said, "lookup ") && strings.Contains(said, "i/o timeout")
+}
+
+// cloneFromForge runs `git clone` of url into dest (git's arguments in args, run from dir), and while it fails on the
+// name lookup, clones again with a growing pause until lookupWait has passed. What a failed try left at dest is
+// removed before the next. Any other failure is returned at once.
+func cloneFromForge(ctx context.Context, run Runner, dir, url, dest string, args []string,
+	say func(step, format string, args ...any)) error {
+	_, err := run(ctx, dir, "git", args...)
+	if !failedLookup(err) {
+		return err
+	}
+	started := time.Now()
+	pause := lookupFirstPause
+	for {
+		elapsed := time.Since(started)
+		left := lookupWait - elapsed
+		if left <= 0 {
+			say("clone", "the resolver did not answer for %s, cloning %s, for %s; the clone fails",
+				urlHost(url), url, elapsed.Round(time.Second))
+			return fmt.Errorf("the resolver did not answer a lookup of %s for %s, cloning %s: %w",
+				urlHost(url), elapsed.Round(time.Millisecond), url, err)
+		}
+		wait := min(pause, left)
+		say("clone", "the resolver did not answer for %s, cloning %s; trying again in %s, for up to %s "+
+			"(novox/hq issue 1773)", urlHost(url), url, wait, lookupWait)
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped while waiting for the resolver to answer for %s: %w (last: %v)",
+				urlHost(url), ctx.Err(), err)
+		case <-time.After(wait):
+		}
+		pause = min(pause*2, lookupMostPause)
+		if err := os.RemoveAll(dest); err != nil {
+			return err
+		}
+		if _, err = run(ctx, dir, "git", args...); !failedLookup(err) {
+			if err == nil {
+				say("clone", "the resolver answered for %s after %s; the clone worked", urlHost(url),
+					time.Since(started).Round(time.Millisecond))
+			}
+			return err
+		}
+	}
+}
+
+// urlHost is the host (and port) a URL names, or the URL itself when it names none.
+func urlHost(url string) string {
+	rest := url
+	if _, after, ok := strings.Cut(url, "://"); ok {
+		rest = after
+	}
+	if i := strings.IndexAny(rest, "/?#"); i >= 0 {
+		rest = rest[:i]
+	}
+	if i := strings.LastIndex(rest, "@"); i >= 0 {
+		rest = rest[i+1:]
+	}
+	if rest == "" {
+		return url
+	}
+	return rest
+}