mesh-controller/internal/builder/registry_wait.go

package builder

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"syscall"
	"time"
)

// A build waits out a registry that refuses connections, for a bounded time (novox/hq issue 457).
//
// **Why waiting, and why here.** The store's nightly collection holds the registry still for its run —
// about a minute and a half, measured on 2026-10-11 — and a build that reached the registry in that
// window failed on "connection refused", and its whole delivery plan with it: a plan failed for a
// pause the mesh itself scheduled. The other design weighed was the collection telling the controller
// it holds the registry, and the controller holding build asks while it runs. Waiting here is smaller
// and covers more: it is local to the one place that talks to the registry, needs no new message
// between modules, and also carries a build over any other short outage — a registry restarted by its
// own update, say. A refusal is the one error waited for: nothing was sent, so trying again cannot
// do anything twice, and it is what a registry that is stopped answers.
//
// **Bounded, and loud past the bound.** registryWait is longer than the collection holds the registry
// (five minutes against about one and a half), so the pause the mesh schedules is always waited out,
// and a registry that is really down still fails the build — saying how long it was refused — rather
// than holding a build machine for ever. Every wait is said in the build's log, with why, and so is
// the registry answering again.

var (
	// registryWait is how long a build waits for a registry that refuses, per call that found it so.
	registryWait = 5 * time.Minute
	// registryFirstPause is the first pause between tries; each pause doubles, up to registryMostPause.
	registryFirstPause = time.Second
)

// registryMostPause is the longest pause between two tries: short enough that a build goes on within
// seconds of the registry answering again.
const registryMostPause = 10 * time.Second

// refused is whether an error is a connection refused: from a dial here, or as a command such as docker
// said it in its output.
func refused(err error) bool {
	return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
}

// waitForRegistry runs try, and while it fails because the registry at address refuses connections,
// tries again with a growing pause until registryWait has passed. what names the call, for the log.
func waitForRegistry(ctx context.Context, address, what string, try func() error) error {
	err := try()
	if !refused(err) {
		return err
	}
	started := time.Now()
	pause := registryFirstPause
	tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+
		"the store's nightly collection runs, about a minute and a half (novox/hq issue 457)",
		what, address, registryWait)
	for {
		left := registryWait - time.Since(started)
		if left <= 0 {
			tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address,
				time.Since(started).Round(time.Second))
			return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+
				"collection holds it still, so it is down, not paused: %w",
				address, time.Since(started).Round(time.Millisecond), err)
		}
		wait := min(pause, left)
		select {
		case <-ctx.Done():
			return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", address, ctx.Err(), err)
		case <-time.After(wait):
		}
		pause = min(pause*2, registryMostPause)
		if err = try(); !refused(err) {
			tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address,
				time.Since(started).Round(time.Millisecond))
			return err
		}
	}
}

// waitingTransport waits for the registry on every request to it, and on none to anywhere else: the
// same client copies from upstream registries, whose refusals are theirs to answer.
type waitingTransport struct {
	base    http.RoundTripper
	address string
}

func (t waitingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
	if request.URL.Host != t.address {
		return t.base.RoundTrip(request)
	}
	var response *http.Response
	tries := 0
	err := waitForRegistry(request.Context(), t.address, request.Method+" "+request.URL.Path, func() error {
		attempt := request
		if tries > 0 && request.Body != nil && request.Body != http.NoBody {
			// A body is sent again only when it can be read again; one that cannot is not retried.
			if request.GetBody == nil {
				return fmt.Errorf("%s %s cannot be sent again: its body cannot be read twice", request.Method, request.URL)
			}
			body, err := request.GetBody()
			if err != nil {
				return err
			}
			attempt = request.Clone(request.Context())
			attempt.Body = body
		}
		tries++
		var err error
		response, err = t.base.RoundTrip(attempt)
		return err
	})
	return response, err
}

// waiting is a client like c whose requests to the registry wait for it.
func waiting(c *http.Client, address string) *http.Client {
	if _, already := c.Transport.(waitingTransport); already {
		return c
	}
	copied := *c
	base := c.Transport
	if base == nil {
		base = http.DefaultTransport
	}
	copied.Transport = waitingTransport{base: base, address: address}
	return &copied
}