C2: mesh-build-go says a failure of the network as a transport fault: one download, no retry of its own (ADR 0664 §1, issue 1748)

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
2026-11-09 23:10:16 +01:00
parent 02d57c4a53
commit b9cfa4fe16
10 changed files with 102 additions and 91 deletions
+13 -2
View File
@@ -9,6 +9,7 @@ import (
"regexp"
"strings"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
dforgekey "git.novox.be/novox/mesh-delivery/sdk/forgekey"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
@@ -52,6 +53,16 @@ var (
refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`)
)
// gitFailure is a failed git command said in git's own words; one whose words are a network's (a lookup, a timeout, ssh's key
// exchange cut by a forge's per-source penalty) is a sdkstep.Passing, so the ask is given again after a delay (ADR 0664 §1).
func gitFailure(verb, said string) error {
err := fmt.Errorf("git %s: %s", verb, said)
if sdkstep.PassingCause(said) {
return &sdkstep.Passing{Err: err}
}
return err
}
// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached
// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh://<ssh name>/<owner>/<repo>.git, the ssh name being the
// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names
@@ -72,7 +83,7 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
r := run(ctx, env, "git", full...)
if r.Status != 0 || r.Err != "" {
return "", fmt.Errorf("git %s: %s", args[0], dforgekey.Said(r.Err+"\n"+r.Stderr))
return "", gitFailure(args[0], dforgekey.Said(r.Err+"\n"+r.Stderr))
}
return r.Stdout, nil
}
@@ -133,7 +144,7 @@ func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, comm
if verb == "-C" && len(args) > 2 { // the mirror's commands are named by what they do, not by -C
verb = args[2]
}
return fmt.Errorf("git %s: %s", verb, dforgekey.Said(r.Err+"\n"+r.Stderr))
return gitFailure(verb, dforgekey.Said(r.Err+"\n"+r.Stderr))
}
return nil
}
@@ -2,6 +2,7 @@ package buildstep
import (
"context"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"strings"
"testing"
@@ -93,6 +94,28 @@ func TestFetchCommitFallsBackToThePullRef(t *testing.T) {
if err := FetchCommit(context.Background(), h.run, "/w", "ssh://mesh-forge/o/r.git", sha, "refs/pull/1/head"); err == nil {
t.Error("a commit that is not in the ref was accepted")
}
// ADR 0664 §1 (issue 1748): the words a forge's sshd penalty leaves (captured on ace, 2026-11-09) are a transport fault the bus
// gives the ask again for; a missing commit is not.
for _, c := range []struct {
said string
passing bool
}{
{"kex_exchange_identification: read: Connection reset by peer\nfatal: Could not read from remote repository.", true},
{"Connection closed by 10.10.0.1 port 222", true},
{"ssh: Could not resolve hostname mesh-forge: Temporary failure in name resolution", true},
{"fatal: couldn't find remote ref " + sha, false},
} {
x := &fakeGit{rule: func(l string) forgekey.Ran {
if strings.Contains(l, " fetch ") {
return forgekey.Ran{Status: 128, Stderr: c.said}
}
return forgekey.Ran{}
}}
if err := FetchCommit(context.Background(), x.run, "/w", "ssh://mesh-forge/o/r.git", sha, ""); err == nil || sdkstep.IsPassing(err) != c.passing {
t.Errorf("%q: %v, passing %v, want %v", c.said, err, err != nil && sdkstep.IsPassing(err), c.passing)
}
}
}
// 1474/AC3: HEAD differs from the commit after the checkout: an error naming both.
+1 -1
View File
@@ -56,7 +56,7 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
r := run(ctx, env, "git", full...)
if r.Status != 0 || r.Err != "" {
return "", fmt.Errorf("git %s: %s", args[0], dforgekey.Said(r.Err+"\n"+r.Stderr))
return "", gitFailure(args[0], dforgekey.Said(r.Err+"\n"+r.Stderr))
}
return r.Stdout, nil
}
@@ -0,0 +1,14 @@
package buildstep
import sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
// Transport is err as a sdkstep.Passing when its words are those of a network that did not answer (a lookup, a timeout, a
// reset connection; sdkstep.PassingCause), so the build module's ask is given again after a delay instead of failing
// (novox/hq ADR 0664 §1, issue 1748); any other error, nil included, is returned as it is. Every build module wraps a step
// that fetches over the network with it: a failure of the source (an unknown revision, a wrong checksum) never is one.
func Transport(err error) error {
if err == nil || sdkstep.IsPassing(err) || !sdkstep.PassingCause(err.Error()) {
return err
}
return &sdkstep.Passing{Err: err}
}
@@ -23,6 +23,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/therootcompany/xz"
)
@@ -145,7 +147,7 @@ func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue
}
response, err := client.Do(request)
if err != nil {
return "", fmt.Errorf("cannot fetch %s: %w", art.URL, err)
return "", buildstep.Transport(fmt.Errorf("cannot fetch %s: %w", art.URL, err))
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
@@ -162,7 +164,7 @@ func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue
err = cerr
}
if err != nil {
return "", fmt.Errorf("fetching %s: %w", art.URL, err)
return "", buildstep.Transport(fmt.Errorf("fetching %s: %w", art.URL, err))
}
if n > maxUpstream {
return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", art.URL, maxUpstream)
@@ -6,6 +6,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net"
"net/http"
@@ -223,8 +224,17 @@ func TestAPinnedFileOfAnotherDigestIsRefused(t *testing.T) {
func TestAPinnedFileTheServerRefusesFailsTheStep(t *testing.T) {
m, states := pinnedModule(t, "/gone", releaseSHA256, `,"file":"x"`)
if _, st := runBuild(t, m, states, "files"); st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "404") {
t.Errorf("the step ended %s: %q", st.State, st.Reason)
if _, st := runBuild(t, m, states, "files"); st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "404") || st.Passing {
t.Errorf("the step ended %s: %q (passing %v): a missing file is no transport fault", st.State, st.Reason, st.Passing)
}
// ADR 0664 §1 (issue 1748): a server that cannot be reached is a transport fault, ended failed as passing for the bus to
// give the ask again.
m, states = pinnedModule(t, "/release.tar.gz", releaseSHA256, `,"file":"x"`)
m.HTTP.Transport.(*http.Transport).DialContext = func(context.Context, string, string) (net.Conn, error) {
return nil, errors.New("dial tcp 192.0.2.1:443: connect: connection refused")
}
if _, st := runBuild(t, m, states, "files"); st.State != sdkstep.StepFailed || !st.Passing {
t.Errorf("an unreachable server ended %s: %q (passing %v)", st.State, st.Reason, st.Passing)
}
}
@@ -43,8 +43,6 @@ type Module struct {
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
Ctx context.Context
Say func(format string, a ...any)
// Pause waits between a download and its retaking (versions.go); nil waits on a timer. A test sets it to record.
Pause func(d time.Duration)
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches,
// both outside the account's home (ADR 0466 §1); Home is the account's home.
Run forgekey.Runner
@@ -150,8 +150,14 @@ func (m *Module) placePin(ctx context.Context, root string, p pin, failure strin
repository := "novox/" + p.repo
r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-c", "credential.helper=", "ls-remote", m.remote(repository), "refs/pull/*/head")
if r.Status != 0 || r.Err != "" {
// The forge did not answer: retaken, with its own cause, never held as a refusal until the module changes (issue 1748).
return "", fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s failed (a download failure that may be retaken): %s", p.module, p.short, repository, dforgekey.Said(r.Err+"\n"+r.Stderr))
// The forge did not answer: its own cause, never held as a refusal until the module changes (issue 1748); a network's
// words make it a sdkstep.Passing, given again after a delay (ADR 0664 §1).
said := dforgekey.Said(r.Err + "\n" + r.Stderr)
err := fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s failed: %s", p.module, p.short, repository, said)
if sdkstep.PassingCause(said) {
err = &sdkstep.Passing{Err: err}
}
return "", err
}
commit, ref := "", ""
for _, l := range strings.Split(r.Stdout, "\n") {
@@ -30,7 +30,6 @@ import (
"strconv"
"strings"
"sync"
"time"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
@@ -97,40 +96,17 @@ func saidSuffix(said string) string {
return ": " + said
}
// downloadAttempts bounds the downloads of one asked version, and downloadBackoff is the wait after the first failed one
// (twice that after the second): a network failure is taken again, any other fails at once.
const (
downloadAttempts = 3
downloadBackoff = 3 * time.Second
)
// networkFailure says go's words are those of a network that did not answer (a name lookup, a timeout, a refused or
// reset connection), not of a version or a checksum that is wrong.
func networkFailure(s string) bool {
s = strings.ToLower(s)
for _, w := range []string{"i/o timeout", "no such host", "temporary failure in name resolution", "connection refused", "connection reset", "tls handshake timeout", "lookup "} {
if strings.Contains(s, w) {
return true
}
}
return false
}
// retryWait waits d (m.Pause in a test) or until ctx ends.
func (m *Module) retryWait(ctx context.Context, d time.Duration) error {
if m.Pause != nil {
m.Pause(d)
return ctx.Err()
}
select {
case <-time.After(d):
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// takeVersion takes one asked version into dir's modfile (takeVersions).
// downloadFailure is a failed download said in go's own words; one whose words are a network's (a lookup, a timeout, a reset
// connection) is a sdkstep.Passing, so the build module gives the ask again after a delay instead of failing it (ADR 0664 §1).
func downloadFailure(failure, format string, args ...any) error {
err := fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), causeLine(failure))
if sdkstep.PassingCause(failure) {
return &sdkstep.Passing{Err: err}
}
return err
}
func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVersion, pins *sync.Mutex) error {
modfile, err := m.ensureModfile(dir)
if err != nil {
@@ -140,32 +116,12 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer
tagged := sdkstep.PackageSemver.MatchString(tv.Asked)
var r forgekey.Ran
var info downloadInfo
var failed bool
attempts := 0
for {
attempts++
r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked)
info = downloadInfo{}
_ = json.Unmarshal([]byte(r.Stdout), &info)
failed = r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != ""
if !failed || attempts >= downloadAttempts || !networkFailure(info.Error+"\n"+r.Err+"\n"+r.Stderr) {
break
}
// A name lookup, a timeout or a refused connection says nothing of the version: the download is taken again.
if m.Say != nil {
m.Say("%s: downloading %s@%s failed on the network (attempt %d of %d), taken again: %s", filepath.Base(dir), tv.Package, tv.Asked, attempts, downloadAttempts, causeLine(info.Error+"\n"+r.Err+"\n"+r.Stderr))
}
if err := m.retryWait(ctx, time.Duration(attempts)*downloadBackoff); err != nil {
return err
}
}
tried := "taken once"
if attempts > 1 {
tried = fmt.Sprintf("taken %d times", attempts)
}
r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked)
_ = json.Unmarshal([]byte(r.Stdout), &info)
failed := r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != ""
if failed && tagged {
failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr)
return fmt.Errorf("downloading the group's tagged version %s of %s through the Go seat failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure))
return downloadFailure(failure, "downloading the group's tagged version %s of %s through the Go seat failed", tv.Asked, tv.Package)
}
if failed {
// Not a commit Go can fetch: the head of a group's pull request (ADR 0573 §4), placed from the forge's heads.
@@ -183,7 +139,7 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer
pins.Unlock()
// On no head, and go failed for another cause (the forge or the resolver did not answer): it fails with go's own
// words, and not as a kind the builder holds until the module changes (issue 1748).
return fmt.Errorf("downloading the asked version %s of %s failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure))
return downloadFailure(failure, "downloading the asked version %s of %s failed", tv.Asked, tv.Package)
}
if err == nil {
err = m.replaceInModfile(dir, tv.Package, where)
@@ -7,7 +7,6 @@ import (
"path/filepath"
"strings"
"testing"
"time"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
@@ -29,7 +28,8 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
built, said, edit string
failsWith string // a download failure that is not an unknown revision: the step fails with go's own words
forgeDown bool // the forge's listing of pull request heads fails too
timeouts, downloads int // downloads that time out before go answers; the downloads asked in all (0: not counted)
passing bool // the step ends failed as a transport fault, which the bus gives again (ADR 0664 §1)
downloads int // the downloads asked in all (0: not counted)
}{
{name: "a trunk commit newer than go.mod's", asked: trunk,
answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`,
@@ -47,12 +47,12 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
{name: "a head go cannot even look up is placed", asked: commitSHA, failing: true,
answer: `{"Path":"` + sdk + `","Error":"git.novox.be/novox/mesh-sdk/go@` + commitSHA + `: dial tcp: lookup git.novox.be on [::1]:53: connection refused"}`,
built: commitSHA, said: "placed from the pull request's head"},
{name: "a lookup that times out is no pin", asked: offHead, failing: true, failsWith: "i/o timeout",
{name: "a lookup that times out is no pin", asked: offHead, failing: true, failsWith: "i/o timeout", passing: true, downloads: 1,
answer: `{"Path":"` + sdk + `","Error":"git.novox.be/novox/mesh-sdk/go@` + offHead + `: dial tcp: lookup git.novox.be on 10.10.0.3:53: read udp 10.10.0.3:42292->10.10.0.3:53: i/o timeout"}`},
// The forge's own listing failing is a failure that may be retaken too, with its own cause (review 3).
{name: "a forge that cannot be listed is retaken", asked: commitSHA, failing: true, forgeDown: true,
failsWith: "unable to access the forge",
answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`},
failsWith: "unable to access the forge", passing: true,
answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`},
// ADR 0607 §3, §5 (design 51 *Tagged versions*): a group's tagged version is fetched by that version and built
// whatever go.mod names; one the Go seat lacks fails naming it, never looked for among the pull request heads.
{name: "a group's tagged version", asked: "v0.1.15-feat-x.2",
@@ -63,22 +63,16 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
built: "v0.1.14-feat-x.2", said: "the group's tagged version", edit: "-require=" + sdk + "@v0.1.14-feat-x.2"},
{name: "a tagged version the Go seat lacks", asked: "v0.1.15-feat-x.3", failing: true, failsWith: "the group's tagged version v0.1.15-feat-x.3",
answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@v0.1.15-feat-x.3: reading https://seat/go/@v/v0.1.15-feat-x.3.info: 404 Not Found"}`},
// Issue 1966: a network failure is taken again a bounded number of times; another failure is not.
{name: "a timeout twice is taken again", asked: trunk, timeouts: 2, downloads: 3,
answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`,
built: "v0.1.15-0.20261101000000-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.15-0.20261101000000-aaaaaaaaaaaa"},
{name: "a timeout that stays fails after three downloads", asked: offHead, failing: true, failsWith: "taken 3 times", downloads: 3,
answer: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`},
// ADR 0664 §1 (issue 1748): one download; a failure of the network is a transport fault the bus gives the ask again for after a
// delay, no retry here; another failure is not one.
{name: "an unknown revision is not taken again", asked: offHead, failing: true, failsWith: "on no pull request head", downloads: 1,
answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`},
{name: "a refused connection is no pin", asked: offHead, failing: true, failsWith: "connection refused",
{name: "a refused connection is no pin", asked: offHead, failing: true, failsWith: "connection refused", passing: true,
answer: `{"Path":"` + sdk + `","Error":"git ls-remote -q origin in /tmp/gopath: dial tcp 10.10.0.1:443: connect: connection refused"}`},
} {
t.Run(c.name, func(t *testing.T) {
h, states := newPinHarness(t, pinnedGoMod, false, pullList)
h.f.gomods = "go.mod\nother/go.mod\n"
var paused []time.Duration
h.m.Pause = func(d time.Duration) { paused = append(paused, d) }
downloads := 0
base := h.m.Run
h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
@@ -103,9 +97,6 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
h.cmds = append(h.cmds, cmdRecord{env: env, line: line})
h.mu.Unlock()
downloads++
if downloads <= c.timeouts {
return forgekey.Ran{Status: 1, Stdout: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`}
}
if c.failing {
return forgekey.Ran{Status: 1, Stdout: c.answer}
}
@@ -118,18 +109,18 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) {
t.Fatal(err)
}
st := states.waitEnd(t)
if c.downloads != 0 && (downloads != c.downloads || len(paused) != c.downloads-1) {
t.Errorf("go downloaded %d times with %d waits, want %d downloads", downloads, len(paused), c.downloads)
if c.downloads != 0 && downloads != c.downloads {
t.Errorf("go downloaded %d times, want %d: no retry here", downloads, c.downloads)
}
if c.failsWith == "on no pull request head" { // not a network failure: the forge's heads said, no retaking
if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) {
if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) || st.Passing {
t.Errorf("the unknown revision ended %s: %q", st.State, st.Reason)
}
return
}
if c.failsWith != "" {
if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) || strings.Contains(st.Reason, "no pull request head") ||
!strings.Contains(st.Reason, "may be retaken") || builds.FailedUntilChanged(st.Reason) {
strings.Contains(st.Reason, "may be retaken") || st.Passing != c.passing || builds.FailedUntilChanged(st.Reason) {
t.Errorf("the download failure ended %s: %q", st.State, st.Reason)
}
return