From 77844477ec0d64eabe95583f65134c4e103ec8e6 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 10:09:35 +0100 Subject: [PATCH] mesh-build-go takes a download again after a network failure, three times with a short wait, and says so (issue 1921, issue 1966) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../mesh-build-go/cmd/mesh-build-go/module.go | 2 + .../cmd/mesh-build-go/versions.go | 66 +++++++++++++++++-- .../cmd/mesh-build-go/versions_test.go | 26 ++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 75bd3cff2..af8e786d4 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -41,6 +41,8 @@ type Module struct { // Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not). Ctx context.Context Say func(format string, a ...any) + // Pause waits between a download and its retaking (versions.go); nil waits on a timer. A test sets it to record. + Pause func(d time.Duration) // Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches, // both outside the account's home (ADR 0466 §1); Home is the account's home. Run forgekey.Runner diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index a57c0f570..4d182addd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -30,8 +30,10 @@ import ( "strconv" "strings" "sync" + "time" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // TakenVersion is what the fetch built a directory against for one asked package: the asked commit, the version its go.mod @@ -95,6 +97,39 @@ func saidSuffix(said string) string { return ": " + said } +// downloadAttempts bounds the downloads of one asked version, and downloadBackoff is the wait after the first failed one +// (twice that after the second): a network failure is taken again, any other fails at once. +const ( + downloadAttempts = 3 + downloadBackoff = 3 * time.Second +) + +// networkFailure says go's words are those of a network that did not answer (a name lookup, a timeout, a refused or +// reset connection), not of a version or a checksum that is wrong. +func networkFailure(s string) bool { + s = strings.ToLower(s) + for _, w := range []string{"i/o timeout", "no such host", "temporary failure in name resolution", "connection refused", "connection reset", "tls handshake timeout", "lookup "} { + if strings.Contains(s, w) { + return true + } + } + return false +} + +// retryWait waits d (m.Pause in a test) or until ctx ends. +func (m *Module) retryWait(ctx context.Context, d time.Duration) error { + if m.Pause != nil { + m.Pause(d) + return ctx.Err() + } + select { + case <-time.After(d): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // takeVersion takes one asked version into dir's modfile (takeVersions). func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVersion, pins *sync.Mutex) error { modfile, err := m.ensureModfile(dir) @@ -103,13 +138,34 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer } pinsDir := filepath.Dir(modfile) tagged := sdkstep.PackageSemver.MatchString(tv.Asked) - r := m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked) + var r forgekey.Ran var info downloadInfo - _ = json.Unmarshal([]byte(r.Stdout), &info) - failed := r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" + var failed bool + attempts := 0 + for { + attempts++ + r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked) + info = downloadInfo{} + _ = json.Unmarshal([]byte(r.Stdout), &info) + failed = r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" + if !failed || attempts >= downloadAttempts || !networkFailure(info.Error+"\n"+r.Err+"\n"+r.Stderr) { + break + } + // A name lookup, a timeout or a refused connection says nothing of the version: the download is taken again. + if m.Say != nil { + m.Say("%s: downloading %s@%s failed on the network (attempt %d of %d), taken again: %s", filepath.Base(dir), tv.Package, tv.Asked, attempts, downloadAttempts, causeLine(info.Error+"\n"+r.Err+"\n"+r.Stderr)) + } + if err := m.retryWait(ctx, time.Duration(attempts)*downloadBackoff); err != nil { + return err + } + } + tried := "taken once" + if attempts > 1 { + tried = fmt.Sprintf("taken %d times", attempts) + } if failed && tagged { failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr) - return fmt.Errorf("downloading the group's tagged version %s of %s through the Go seat failed (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure)) + return fmt.Errorf("downloading the group's tagged version %s of %s through the Go seat failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure)) } if failed { // Not a commit Go can fetch: the head of a group's pull request (ADR 0573 §4), placed from the forge's heads. @@ -127,7 +183,7 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer pins.Unlock() // On no head, and go failed for another cause (the forge or the resolver did not answer): it fails with go's own // words, and not as a kind the builder holds until the module changes (issue 1748). - return fmt.Errorf("downloading the asked version %s of %s failed (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure)) + return fmt.Errorf("downloading the asked version %s of %s failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure)) } if err == nil { err = m.replaceInModfile(dir, tv.Package, where) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go index 958a5a34c..e24c6415b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "time" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -28,6 +29,7 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { built, said, edit string failsWith string // a download failure that is not an unknown revision: the step fails with go's own words forgeDown bool // the forge's listing of pull request heads fails too + timeouts, downloads int // downloads that time out before go answers; the downloads asked in all (0: not counted) }{ {name: "a trunk commit newer than go.mod's", asked: trunk, answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`, @@ -61,12 +63,23 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { built: "v0.1.14-feat-x.2", said: "the group's tagged version", edit: "-require=" + sdk + "@v0.1.14-feat-x.2"}, {name: "a tagged version the Go seat lacks", asked: "v0.1.15-feat-x.3", failing: true, failsWith: "the group's tagged version v0.1.15-feat-x.3", answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@v0.1.15-feat-x.3: reading https://seat/go/@v/v0.1.15-feat-x.3.info: 404 Not Found"}`}, + // Issue 1966: a network failure is taken again a bounded number of times; another failure is not. + {name: "a timeout twice is taken again", asked: trunk, timeouts: 2, downloads: 3, + answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`, + built: "v0.1.15-0.20261101000000-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.15-0.20261101000000-aaaaaaaaaaaa"}, + {name: "a timeout that stays fails after three downloads", asked: offHead, failing: true, failsWith: "taken 3 times", downloads: 3, + answer: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`}, + {name: "an unknown revision is not taken again", asked: offHead, failing: true, failsWith: "on no pull request head", downloads: 1, + answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`}, {name: "a refused connection is no pin", asked: offHead, failing: true, failsWith: "connection refused", answer: `{"Path":"` + sdk + `","Error":"git ls-remote -q origin in /tmp/gopath: dial tcp 10.10.0.1:443: connect: connection refused"}`}, } { t.Run(c.name, func(t *testing.T) { h, states := newPinHarness(t, pinnedGoMod, false, pullList) h.f.gomods = "go.mod\nother/go.mod\n" + var paused []time.Duration + h.m.Pause = func(d time.Duration) { paused = append(paused, d) } + downloads := 0 base := h.m.Run h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { line := name + " " + strings.Join(args, " ") @@ -89,6 +102,10 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { h.mu.Lock() h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) h.mu.Unlock() + downloads++ + if downloads <= c.timeouts { + return forgekey.Ran{Status: 1, Stdout: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`} + } if c.failing { return forgekey.Ran{Status: 1, Stdout: c.answer} } @@ -101,6 +118,15 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { t.Fatal(err) } st := states.waitEnd(t) + if c.downloads != 0 && (downloads != c.downloads || len(paused) != c.downloads-1) { + t.Errorf("go downloaded %d times with %d waits, want %d downloads", downloads, len(paused), c.downloads) + } + if c.failsWith == "on no pull request head" { // not a network failure: the forge's heads said, no retaking + if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) { + t.Errorf("the unknown revision ended %s: %q", st.State, st.Reason) + } + return + } if c.failsWith != "" { if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) || strings.Contains(st.Reason, "no pull request head") || !strings.Contains(st.Reason, "may be retaken") || builds.FailedUntilChanged(st.Reason) {