mesh-build-go takes a download again after a network failure, three times with a short wait, and says so (issue 1921, issue 1966)

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i

modules/mesh-build-go/cmd/mesh-build-go/module.go

@@ -41,6 +41,8 @@ type Module struct {
 	// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
 	Ctx context.Context
 	Say func(format string, a ...any)
+	// Pause waits between a download and its retaking (versions.go); nil waits on a timer. A test sets it to record.
+	Pause func(d time.Duration)
 	// Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches,
 	// both outside the account's home (ADR 0466 §1); Home is the account's home.
 	Run               forgekey.Runner

modules/mesh-build-go/cmd/mesh-build-go/versions.go

@@ -30,8 +30,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
 )
 
 // TakenVersion is what the fetch built a directory against for one asked package: the asked commit, the version its go.mod
@@ -95,6 +97,39 @@ func saidSuffix(said string) string {
 	return ": " + said
 }
 
+// downloadAttempts bounds the downloads of one asked version, and downloadBackoff is the wait after the first failed one
+// (twice that after the second): a network failure is taken again, any other fails at once.
+const (
+	downloadAttempts = 3
+	downloadBackoff  = 3 * time.Second
+)
+
+// networkFailure says go's words are those of a network that did not answer (a name lookup, a timeout, a refused or
+// reset connection), not of a version or a checksum that is wrong.
+func networkFailure(s string) bool {
+	s = strings.ToLower(s)
+	for _, w := range []string{"i/o timeout", "no such host", "temporary failure in name resolution", "connection refused", "connection reset", "tls handshake timeout", "lookup "} {
+		if strings.Contains(s, w) {
+			return true
+		}
+	}
+	return false
+}
+
+// retryWait waits d (m.Pause in a test) or until ctx ends.
+func (m *Module) retryWait(ctx context.Context, d time.Duration) error {
+	if m.Pause != nil {
+		m.Pause(d)
+		return ctx.Err()
+	}
+	select {
+	case <-time.After(d):
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
 // takeVersion takes one asked version into dir's modfile (takeVersions).
 func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVersion, pins *sync.Mutex) error {
 	modfile, err := m.ensureModfile(dir)
@@ -103,13 +138,34 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer
 	}
 	pinsDir := filepath.Dir(modfile)
 	tagged := sdkstep.PackageSemver.MatchString(tv.Asked)
-	r := m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked)
+	var r forgekey.Ran
 	var info downloadInfo
-	_ = json.Unmarshal([]byte(r.Stdout), &info)
-	failed := r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != ""
+	var failed bool
+	attempts := 0
+	for {
+		attempts++
+		r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked)
+		info = downloadInfo{}
+		_ = json.Unmarshal([]byte(r.Stdout), &info)
+		failed = r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != ""
+		if !failed || attempts >= downloadAttempts || !networkFailure(info.Error+"\n"+r.Err+"\n"+r.Stderr) {
+			break
+		}
+		// A name lookup, a timeout or a refused connection says nothing of the version: the download is taken again.
+		if m.Say != nil {
+			m.Say("%s: downloading %s@%s failed on the network (attempt %d of %d), taken again: %s", filepath.Base(dir), tv.Package, tv.Asked, attempts, downloadAttempts, causeLine(info.Error+"\n"+r.Err+"\n"+r.Stderr))
+		}
+		if err := m.retryWait(ctx, time.Duration(attempts)*downloadBackoff); err != nil {
+			return err
+		}
+	}
+	tried := "taken once"
+	if attempts > 1 {
+		tried = fmt.Sprintf("taken %d times", attempts)
+	}
 	if failed && tagged {
 		failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr)
-		return fmt.Errorf("downloading the group's tagged version %s of %s through the Go seat failed (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure))
+		return fmt.Errorf("downloading the group's tagged version %s of %s through the Go seat failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure))
 	}
 	if failed {
 		// Not a commit Go can fetch: the head of a group's pull request (ADR 0573 §4), placed from the forge's heads.
@@ -127,7 +183,7 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer
 			pins.Unlock()
 			// On no head, and go failed for another cause (the forge or the resolver did not answer): it fails with go's own
 			// words, and not as a kind the builder holds until the module changes (issue 1748).
-			return fmt.Errorf("downloading the asked version %s of %s failed (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure))
+			return fmt.Errorf("downloading the asked version %s of %s failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure))
 		}
 		if err == nil {
 			err = m.replaceInModfile(dir, tv.Package, where)

modules/mesh-build-go/cmd/mesh-build-go/versions_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
 	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
@@ -28,6 +29,7 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
 		built, said, edit   string
 		failsWith           string // a download failure that is not an unknown revision: the step fails with go's own words
 		forgeDown           bool   // the forge's listing of pull request heads fails too
+		timeouts, downloads int    // downloads that time out before go answers; the downloads asked in all (0: not counted)
 	}{
 		{name: "a trunk commit newer than go.mod's", asked: trunk,
 			answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`,
@@ -61,12 +63,23 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
 			built:  "v0.1.14-feat-x.2", said: "the group's tagged version", edit: "-require=" + sdk + "@v0.1.14-feat-x.2"},
 		{name: "a tagged version the Go seat lacks", asked: "v0.1.15-feat-x.3", failing: true, failsWith: "the group's tagged version v0.1.15-feat-x.3",
 			answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@v0.1.15-feat-x.3: reading https://seat/go/@v/v0.1.15-feat-x.3.info: 404 Not Found"}`},
+		// Issue 1966: a network failure is taken again a bounded number of times; another failure is not.
+		{name: "a timeout twice is taken again", asked: trunk, timeouts: 2, downloads: 3,
+			answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`,
+			built:  "v0.1.15-0.20261101000000-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.15-0.20261101000000-aaaaaaaaaaaa"},
+		{name: "a timeout that stays fails after three downloads", asked: offHead, failing: true, failsWith: "taken 3 times", downloads: 3,
+			answer: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`},
+		{name: "an unknown revision is not taken again", asked: offHead, failing: true, failsWith: "on no pull request head", downloads: 1,
+			answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`},
 		{name: "a refused connection is no pin", asked: offHead, failing: true, failsWith: "connection refused",
 			answer: `{"Path":"` + sdk + `","Error":"git ls-remote -q origin in /tmp/gopath: dial tcp 10.10.0.1:443: connect: connection refused"}`},
 	} {
 		t.Run(c.name, func(t *testing.T) {
 			h, states := newPinHarness(t, pinnedGoMod, false, pullList)
 			h.f.gomods = "go.mod\nother/go.mod\n"
+			var paused []time.Duration
+			h.m.Pause = func(d time.Duration) { paused = append(paused, d) }
+			downloads := 0
 			base := h.m.Run
 			h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
 				line := name + " " + strings.Join(args, " ")
@@ -89,6 +102,10 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
 					h.mu.Lock()
 					h.cmds = append(h.cmds, cmdRecord{env: env, line: line})
 					h.mu.Unlock()
+					downloads++
+					if downloads <= c.timeouts {
+						return forgekey.Ran{Status: 1, Stdout: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`}
+					}
 					if c.failing {
 						return forgekey.Ran{Status: 1, Stdout: c.answer}
 					}
@@ -101,6 +118,15 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
 				t.Fatal(err)
 			}
 			st := states.waitEnd(t)
+			if c.downloads != 0 && (downloads != c.downloads || len(paused) != c.downloads-1) {
+				t.Errorf("go downloaded %d times with %d waits, want %d downloads", downloads, len(paused), c.downloads)
+			}
+			if c.failsWith == "on no pull request head" { // not a network failure: the forge's heads said, no retaking
+				if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) {
+					t.Errorf("the unknown revision ended %s: %q", st.State, st.Reason)
+				}
+				return
+			}
 			if c.failsWith != "" {
 				if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) || strings.Contains(st.Reason, "no pull request head") ||
 					!strings.Contains(st.Reason, "may be retaken") || builds.FailedUntilChanged(st.Reason) {