package builder import ( "context" "errors" "fmt" "net/http" "strings" "syscall" "time" ) // A build waits out a registry that refuses connections, for a bounded time (novox/hq issue 457). // // **Why waiting, and why here.** The store's nightly collection holds the registry still for its run — // about a minute and a half, measured on 2026-10-11 — and a build that reached the registry in that // window failed on "connection refused", and its whole delivery plan with it: a plan failed for a // pause the mesh itself scheduled. The other design weighed was the collection telling the controller // it holds the registry, and the controller holding build asks while it runs. Waiting here is smaller // and covers more: it is local to the one place that talks to the registry, needs no new message // between modules, and also carries a build over any other short outage — a registry restarted by its // own update, say. A refusal is the one error waited for: nothing was sent, so trying again cannot // do anything twice, and it is what a registry that is stopped answers. // // **Bounded, and loud past the bound.** registryWait is longer than the collection holds the registry // (five minutes against about one and a half), so the pause the mesh schedules is always waited out, // and a registry that is really down still fails the build — saying how long it was refused — rather // than holding a build machine for ever. Every wait is said in the build's log, with why, and so is // the registry answering again. var ( // registryWait is how long a build waits for a registry that refuses, per call that found it so. registryWait = 5 * time.Minute // registryFirstPause is the first pause between tries; each pause doubles, up to registryMostPause. registryFirstPause = time.Second ) // registryMostPause is the longest pause between two tries: short enough that a build goes on within // seconds of the registry answering again. const registryMostPause = 10 * time.Second // refused is whether an error is a connection refused: from a dial here, or as a command such as docker // said it in its output. func refused(err error) bool { return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) } // waitForRegistry runs try, and while it fails because the registry at address refuses connections, // tries again with a growing pause until registryWait has passed. what names the call, for the log. func waitForRegistry(ctx context.Context, address, what string, try func() error) error { err := try() if !refused(err) { return err } started := time.Now() pause := registryFirstPause tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+ "the store's nightly collection runs, about a minute and a half (novox/hq issue 457)", what, address, registryWait) for { left := registryWait - time.Since(started) if left <= 0 { tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address, time.Since(started).Round(time.Second)) return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+ "collection holds it still, so it is down, not paused: %w", address, time.Since(started).Round(time.Millisecond), err) } wait := min(pause, left) select { case <-ctx.Done(): return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", address, ctx.Err(), err) case <-time.After(wait): } pause = min(pause*2, registryMostPause) if err = try(); !refused(err) { // Said as it is: the registry answering is only the build going on when the call worked. if err == nil { tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address, time.Since(started).Round(time.Millisecond)) } else { tell("registry", "%s: the registry at %s no longer refuses after %s, and answered with: %v", what, address, time.Since(started).Round(time.Millisecond), err) } return err } } } // waitingTransport waits for the registry on every request to it, and on none to anywhere else: the // same client copies from upstream registries, whose refusals are theirs to answer. type waitingTransport struct { base http.RoundTripper address string } func (t waitingTransport) RoundTrip(request *http.Request) (*http.Response, error) { if request.URL.Host != t.address { return t.base.RoundTrip(request) } var response *http.Response tries := 0 err := waitForRegistry(request.Context(), t.address, request.Method+" "+request.URL.Path, func() error { attempt := request if tries > 0 && request.Body != nil && request.Body != http.NoBody { // A body is sent again only when it can be read again; one that cannot is not retried. if request.GetBody == nil { return fmt.Errorf("%s %s cannot be sent again: its body cannot be read twice", request.Method, request.URL) } body, err := request.GetBody() if err != nil { return err } attempt = request.Clone(request.Context()) attempt.Body = body } tries++ var err error response, err = t.base.RoundTrip(attempt) return err }) return response, err } // waiting is a client like c whose requests to the registry wait for it. func waiting(c *http.Client, address string) *http.Client { if _, already := c.Transport.(waitingTransport); already { return c } copied := *c base := c.Transport if base == nil { base = http.DefaultTransport } copied.Transport = waitingTransport{base: base, address: address} return &copied }