Wait out a registry held still, for up to five minutes, instead of failing the build (issue 457)

The store's nightly collection stops the registry for about a minute and a
half; builds in that window failed on connection refused and failed their
whole plans. A refusal is now waited for with a growing pause, said in the
build's log, and still fails the build past the bound.

internal/builder/registry_wait.go

@@ -0,0 +1,131 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// A build waits out a registry that refuses connections, for a bounded time (novox/hq issue 457).
+//
+// **Why waiting, and why here.** The store's nightly collection holds the registry still for its run —
+// about a minute and a half, measured on 2026-10-11 — and a build that reached the registry in that
+// window failed on "connection refused", and its whole delivery plan with it: a plan failed for a
+// pause the mesh itself scheduled. The other design weighed was the collection telling the controller
+// it holds the registry, and the controller holding build asks while it runs. Waiting here is smaller
+// and covers more: it is local to the one place that talks to the registry, needs no new message
+// between modules, and also carries a build over any other short outage — a registry restarted by its
+// own update, say. A refusal is the one error waited for: nothing was sent, so trying again cannot
+// do anything twice, and it is what a registry that is stopped answers.
+//
+// **Bounded, and loud past the bound.** registryWait is longer than the collection holds the registry
+// (five minutes against about one and a half), so the pause the mesh schedules is always waited out,
+// and a registry that is really down still fails the build — saying how long it was refused — rather
+// than holding a build machine for ever. Every wait is said in the build's log, with why, and so is
+// the registry answering again.
+
+var (
+	// registryWait is how long a build waits for a registry that refuses, per call that found it so.
+	registryWait = 5 * time.Minute
+	// registryFirstPause is the first pause between tries; each pause doubles, up to registryMostPause.
+	registryFirstPause = time.Second
+)
+
+// registryMostPause is the longest pause between two tries: short enough that a build goes on within
+// seconds of the registry answering again.
+const registryMostPause = 10 * time.Second
+
+// refused is whether an error is a connection refused: from a dial here, or as a command such as docker
+// said it in its output.
+func refused(err error) bool {
+	return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
+}
+
+// waitForRegistry runs try, and while it fails because the registry at address refuses connections,
+// tries again with a growing pause until registryWait has passed. what names the call, for the log.
+func waitForRegistry(ctx context.Context, address, what string, try func() error) error {
+	err := try()
+	if !refused(err) {
+		return err
+	}
+	started := time.Now()
+	pause := registryFirstPause
+	tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+
+		"the store's nightly collection runs, about a minute and a half (novox/hq issue 457)",
+		what, address, registryWait)
+	for {
+		left := registryWait - time.Since(started)
+		if left <= 0 {
+			tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address,
+				time.Since(started).Round(time.Second))
+			return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+
+				"collection holds it still, so it is down, not paused: %w",
+				address, time.Since(started).Round(time.Millisecond), err)
+		}
+		wait := min(pause, left)
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", address, ctx.Err(), err)
+		case <-time.After(wait):
+		}
+		pause = min(pause*2, registryMostPause)
+		if err = try(); !refused(err) {
+			tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address,
+				time.Since(started).Round(time.Millisecond))
+			return err
+		}
+	}
+}
+
+// waitingTransport waits for the registry on every request to it, and on none to anywhere else: the
+// same client copies from upstream registries, whose refusals are theirs to answer.
+type waitingTransport struct {
+	base    http.RoundTripper
+	address string
+}
+
+func (t waitingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+	if request.URL.Host != t.address {
+		return t.base.RoundTrip(request)
+	}
+	var response *http.Response
+	tries := 0
+	err := waitForRegistry(request.Context(), t.address, request.Method+" "+request.URL.Path, func() error {
+		attempt := request
+		if tries > 0 && request.Body != nil && request.Body != http.NoBody {
+			// A body is sent again only when it can be read again; one that cannot is not retried.
+			if request.GetBody == nil {
+				return fmt.Errorf("%s %s cannot be sent again: its body cannot be read twice", request.Method, request.URL)
+			}
+			body, err := request.GetBody()
+			if err != nil {
+				return err
+			}
+			attempt = request.Clone(request.Context())
+			attempt.Body = body
+		}
+		tries++
+		var err error
+		response, err = t.base.RoundTrip(attempt)
+		return err
+	})
+	return response, err
+}
+
+// waiting is a client like c whose requests to the registry wait for it.
+func waiting(c *http.Client, address string) *http.Client {
+	if _, already := c.Transport.(waitingTransport); already {
+		return c
+	}
+	copied := *c
+	base := c.Transport
+	if base == nil {
+		base = http.DefaultTransport
+	}
+	copied.Transport = waitingTransport{base: base, address: address}
+	return &copied
+}