diff --git a/modules/mesh-build-go/buildstep/fetch.go b/modules/mesh-build-go/buildstep/fetch.go index c61150f2f..fb8d53a6f 100644 --- a/modules/mesh-build-go/buildstep/fetch.go +++ b/modules/mesh-build-go/buildstep/fetch.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" dforgekey "git.novox.be/novox/mesh-delivery/sdk/forgekey" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -52,6 +53,16 @@ var ( refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`) ) +// gitFailure is a failed git command said in git's own words; one whose words are a network's (a lookup, a timeout, ssh's key +// exchange cut by a forge's per-source penalty) is a sdkstep.Passing, so the ask is given again after a delay (ADR 0664 §1). +func gitFailure(verb, said string) error { + err := fmt.Errorf("git %s: %s", verb, said) + if sdkstep.PassingCause(said) { + return &sdkstep.Passing{Err: err} + } + return err +} + // FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached // (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh:////.git, the ssh name being the // one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names @@ -72,7 +83,7 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) r := run(ctx, env, "git", full...) if r.Status != 0 || r.Err != "" { - return "", fmt.Errorf("git %s: %s", args[0], dforgekey.Said(r.Err+"\n"+r.Stderr)) + return "", gitFailure(args[0], dforgekey.Said(r.Err+"\n"+r.Stderr)) } return r.Stdout, nil } @@ -133,7 +144,7 @@ func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, comm if verb == "-C" && len(args) > 2 { // the mirror's commands are named by what they do, not by -C verb = args[2] } - return fmt.Errorf("git %s: %s", verb, dforgekey.Said(r.Err+"\n"+r.Stderr)) + return gitFailure(verb, dforgekey.Said(r.Err+"\n"+r.Stderr)) } return nil } diff --git a/modules/mesh-build-go/buildstep/fetch_test.go b/modules/mesh-build-go/buildstep/fetch_test.go index b989ab6d5..eb2b916c1 100644 --- a/modules/mesh-build-go/buildstep/fetch_test.go +++ b/modules/mesh-build-go/buildstep/fetch_test.go @@ -2,6 +2,7 @@ package buildstep import ( "context" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "strings" "testing" @@ -93,6 +94,28 @@ func TestFetchCommitFallsBackToThePullRef(t *testing.T) { if err := FetchCommit(context.Background(), h.run, "/w", "ssh://mesh-forge/o/r.git", sha, "refs/pull/1/head"); err == nil { t.Error("a commit that is not in the ref was accepted") } + + // ADR 0664 §1 (issue 1748): the words a forge's sshd penalty leaves (captured on ace, 2026-11-09) are a transport fault the bus + // gives the ask again for; a missing commit is not. + for _, c := range []struct { + said string + passing bool + }{ + {"kex_exchange_identification: read: Connection reset by peer\nfatal: Could not read from remote repository.", true}, + {"Connection closed by 10.10.0.1 port 222", true}, + {"ssh: Could not resolve hostname mesh-forge: Temporary failure in name resolution", true}, + {"fatal: couldn't find remote ref " + sha, false}, + } { + x := &fakeGit{rule: func(l string) forgekey.Ran { + if strings.Contains(l, " fetch ") { + return forgekey.Ran{Status: 128, Stderr: c.said} + } + return forgekey.Ran{} + }} + if err := FetchCommit(context.Background(), x.run, "/w", "ssh://mesh-forge/o/r.git", sha, ""); err == nil || sdkstep.IsPassing(err) != c.passing { + t.Errorf("%q: %v, passing %v, want %v", c.said, err, err != nil && sdkstep.IsPassing(err), c.passing) + } + } } // 1474/AC3: HEAD differs from the commit after the checkout: an error naming both. diff --git a/modules/mesh-build-go/buildstep/merge.go b/modules/mesh-build-go/buildstep/merge.go index 958750c62..93fb6b6f9 100644 --- a/modules/mesh-build-go/buildstep/merge.go +++ b/modules/mesh-build-go/buildstep/merge.go @@ -56,7 +56,7 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) r := run(ctx, env, "git", full...) if r.Status != 0 || r.Err != "" { - return "", fmt.Errorf("git %s: %s", args[0], dforgekey.Said(r.Err+"\n"+r.Stderr)) + return "", gitFailure(args[0], dforgekey.Said(r.Err+"\n"+r.Stderr)) } return r.Stdout, nil } diff --git a/modules/mesh-build-go/buildstep/transport.go b/modules/mesh-build-go/buildstep/transport.go new file mode 100644 index 000000000..c5610fa5f --- /dev/null +++ b/modules/mesh-build-go/buildstep/transport.go @@ -0,0 +1,14 @@ +package buildstep + +import sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + +// Transport is err as a sdkstep.Passing when its words are those of a network that did not answer (a lookup, a timeout, a +// reset connection; sdkstep.PassingCause), so the build module's ask is given again after a delay instead of failing +// (novox/hq ADR 0664 §1, issue 1748); any other error, nil included, is returned as it is. Every build module wraps a step +// that fetches over the network with it: a failure of the source (an unknown revision, a wrong checksum) never is one. +func Transport(err error) error { + if err == nil || sdkstep.IsPassing(err) || !sdkstep.PassingCause(err.Error()) { + return err + } + return &sdkstep.Passing{Err: err} +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/archive.go b/modules/mesh-build-go/cmd/mesh-build-go/archive.go index 5c1388153..b6b20191a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/archive.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/archive.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/therootcompany/xz" ) @@ -145,7 +147,7 @@ func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue } response, err := client.Do(request) if err != nil { - return "", fmt.Errorf("cannot fetch %s: %w", art.URL, err) + return "", buildstep.Transport(fmt.Errorf("cannot fetch %s: %w", art.URL, err)) } defer response.Body.Close() if response.StatusCode != http.StatusOK { @@ -162,7 +164,7 @@ func (m *Module) pinnedFile(ctx context.Context, workspace string, art catalogue err = cerr } if err != nil { - return "", fmt.Errorf("fetching %s: %w", art.URL, err) + return "", buildstep.Transport(fmt.Errorf("fetching %s: %w", art.URL, err)) } if n > maxUpstream { return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", art.URL, maxUpstream) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/archive_test.go b/modules/mesh-build-go/cmd/mesh-build-go/archive_test.go index 9df531908..e4752429c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/archive_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/archive_test.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "io" "net" "net/http" @@ -223,8 +224,17 @@ func TestAPinnedFileOfAnotherDigestIsRefused(t *testing.T) { func TestAPinnedFileTheServerRefusesFailsTheStep(t *testing.T) { m, states := pinnedModule(t, "/gone", releaseSHA256, `,"file":"x"`) - if _, st := runBuild(t, m, states, "files"); st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "404") { - t.Errorf("the step ended %s: %q", st.State, st.Reason) + if _, st := runBuild(t, m, states, "files"); st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "404") || st.Passing { + t.Errorf("the step ended %s: %q (passing %v): a missing file is no transport fault", st.State, st.Reason, st.Passing) + } + // ADR 0664 §1 (issue 1748): a server that cannot be reached is a transport fault, ended failed as passing for the bus to + // give the ask again. + m, states = pinnedModule(t, "/release.tar.gz", releaseSHA256, `,"file":"x"`) + m.HTTP.Transport.(*http.Transport).DialContext = func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("dial tcp 192.0.2.1:443: connect: connection refused") + } + if _, st := runBuild(t, m, states, "files"); st.State != sdkstep.StepFailed || !st.Passing { + t.Errorf("an unreachable server ended %s: %q (passing %v)", st.State, st.Reason, st.Passing) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 10bcb61f4..9b2eae2aa 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -43,8 +43,6 @@ type Module struct { // Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not). Ctx context.Context Say func(format string, a ...any) - // Pause waits between a download and its retaking (versions.go); nil waits on a timer. A test sets it to record. - Pause func(d time.Duration) // Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches, // both outside the account's home (ADR 0466 §1); Home is the account's home. Run forgekey.Runner diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins.go b/modules/mesh-build-go/cmd/mesh-build-go/pins.go index 890d232b7..8563ba96a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins.go @@ -150,8 +150,14 @@ func (m *Module) placePin(ctx context.Context, root string, p pin, failure strin repository := "novox/" + p.repo r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-c", "credential.helper=", "ls-remote", m.remote(repository), "refs/pull/*/head") if r.Status != 0 || r.Err != "" { - // The forge did not answer: retaken, with its own cause, never held as a refusal until the module changes (issue 1748). - return "", fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s failed (a download failure that may be retaken): %s", p.module, p.short, repository, dforgekey.Said(r.Err+"\n"+r.Stderr)) + // The forge did not answer: its own cause, never held as a refusal until the module changes (issue 1748); a network's + // words make it a sdkstep.Passing, given again after a delay (ADR 0664 §1). + said := dforgekey.Said(r.Err + "\n" + r.Stderr) + err := fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s failed: %s", p.module, p.short, repository, said) + if sdkstep.PassingCause(said) { + err = &sdkstep.Passing{Err: err} + } + return "", err } commit, ref := "", "" for _, l := range strings.Split(r.Stdout, "\n") { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index 4d182addd..48eff74e5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -30,7 +30,6 @@ import ( "strconv" "strings" "sync" - "time" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/forgekey" @@ -97,40 +96,17 @@ func saidSuffix(said string) string { return ": " + said } -// downloadAttempts bounds the downloads of one asked version, and downloadBackoff is the wait after the first failed one -// (twice that after the second): a network failure is taken again, any other fails at once. -const ( - downloadAttempts = 3 - downloadBackoff = 3 * time.Second -) - -// networkFailure says go's words are those of a network that did not answer (a name lookup, a timeout, a refused or -// reset connection), not of a version or a checksum that is wrong. -func networkFailure(s string) bool { - s = strings.ToLower(s) - for _, w := range []string{"i/o timeout", "no such host", "temporary failure in name resolution", "connection refused", "connection reset", "tls handshake timeout", "lookup "} { - if strings.Contains(s, w) { - return true - } - } - return false -} - -// retryWait waits d (m.Pause in a test) or until ctx ends. -func (m *Module) retryWait(ctx context.Context, d time.Duration) error { - if m.Pause != nil { - m.Pause(d) - return ctx.Err() - } - select { - case <-time.After(d): - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - // takeVersion takes one asked version into dir's modfile (takeVersions). +// downloadFailure is a failed download said in go's own words; one whose words are a network's (a lookup, a timeout, a reset +// connection) is a sdkstep.Passing, so the build module gives the ask again after a delay instead of failing it (ADR 0664 §1). +func downloadFailure(failure, format string, args ...any) error { + err := fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), causeLine(failure)) + if sdkstep.PassingCause(failure) { + return &sdkstep.Passing{Err: err} + } + return err +} + func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVersion, pins *sync.Mutex) error { modfile, err := m.ensureModfile(dir) if err != nil { @@ -140,32 +116,12 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer tagged := sdkstep.PackageSemver.MatchString(tv.Asked) var r forgekey.Ran var info downloadInfo - var failed bool - attempts := 0 - for { - attempts++ - r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked) - info = downloadInfo{} - _ = json.Unmarshal([]byte(r.Stdout), &info) - failed = r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" - if !failed || attempts >= downloadAttempts || !networkFailure(info.Error+"\n"+r.Err+"\n"+r.Stderr) { - break - } - // A name lookup, a timeout or a refused connection says nothing of the version: the download is taken again. - if m.Say != nil { - m.Say("%s: downloading %s@%s failed on the network (attempt %d of %d), taken again: %s", filepath.Base(dir), tv.Package, tv.Asked, attempts, downloadAttempts, causeLine(info.Error+"\n"+r.Err+"\n"+r.Stderr)) - } - if err := m.retryWait(ctx, time.Duration(attempts)*downloadBackoff); err != nil { - return err - } - } - tried := "taken once" - if attempts > 1 { - tried = fmt.Sprintf("taken %d times", attempts) - } + r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked) + _ = json.Unmarshal([]byte(r.Stdout), &info) + failed := r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" if failed && tagged { failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr) - return fmt.Errorf("downloading the group's tagged version %s of %s through the Go seat failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure)) + return downloadFailure(failure, "downloading the group's tagged version %s of %s through the Go seat failed", tv.Asked, tv.Package) } if failed { // Not a commit Go can fetch: the head of a group's pull request (ADR 0573 §4), placed from the forge's heads. @@ -183,7 +139,7 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer pins.Unlock() // On no head, and go failed for another cause (the forge or the resolver did not answer): it fails with go's own // words, and not as a kind the builder holds until the module changes (issue 1748). - return fmt.Errorf("downloading the asked version %s of %s failed (a download failure, %s, that may be retaken by a new ask of this fingerprint): %s", tv.Asked, tv.Package, tried, causeLine(failure)) + return downloadFailure(failure, "downloading the asked version %s of %s failed", tv.Asked, tv.Package) } if err == nil { err = m.replaceInModfile(dir, tv.Package, where) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go index e24c6415b..68097ae6f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "strings" "testing" - "time" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -29,7 +28,8 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { built, said, edit string failsWith string // a download failure that is not an unknown revision: the step fails with go's own words forgeDown bool // the forge's listing of pull request heads fails too - timeouts, downloads int // downloads that time out before go answers; the downloads asked in all (0: not counted) + passing bool // the step ends failed as a transport fault, which the bus gives again (ADR 0664 §1) + downloads int // the downloads asked in all (0: not counted) }{ {name: "a trunk commit newer than go.mod's", asked: trunk, answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`, @@ -47,12 +47,12 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { {name: "a head go cannot even look up is placed", asked: commitSHA, failing: true, answer: `{"Path":"` + sdk + `","Error":"git.novox.be/novox/mesh-sdk/go@` + commitSHA + `: dial tcp: lookup git.novox.be on [::1]:53: connection refused"}`, built: commitSHA, said: "placed from the pull request's head"}, - {name: "a lookup that times out is no pin", asked: offHead, failing: true, failsWith: "i/o timeout", + {name: "a lookup that times out is no pin", asked: offHead, failing: true, failsWith: "i/o timeout", passing: true, downloads: 1, answer: `{"Path":"` + sdk + `","Error":"git.novox.be/novox/mesh-sdk/go@` + offHead + `: dial tcp: lookup git.novox.be on 10.10.0.3:53: read udp 10.10.0.3:42292->10.10.0.3:53: i/o timeout"}`}, // The forge's own listing failing is a failure that may be retaken too, with its own cause (review 3). {name: "a forge that cannot be listed is retaken", asked: commitSHA, failing: true, forgeDown: true, - failsWith: "unable to access the forge", - answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`}, + failsWith: "unable to access the forge", passing: true, + answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`}, // ADR 0607 §3, §5 (design 51 *Tagged versions*): a group's tagged version is fetched by that version and built // whatever go.mod names; one the Go seat lacks fails naming it, never looked for among the pull request heads. {name: "a group's tagged version", asked: "v0.1.15-feat-x.2", @@ -63,22 +63,16 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { built: "v0.1.14-feat-x.2", said: "the group's tagged version", edit: "-require=" + sdk + "@v0.1.14-feat-x.2"}, {name: "a tagged version the Go seat lacks", asked: "v0.1.15-feat-x.3", failing: true, failsWith: "the group's tagged version v0.1.15-feat-x.3", answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@v0.1.15-feat-x.3: reading https://seat/go/@v/v0.1.15-feat-x.3.info: 404 Not Found"}`}, - // Issue 1966: a network failure is taken again a bounded number of times; another failure is not. - {name: "a timeout twice is taken again", asked: trunk, timeouts: 2, downloads: 3, - answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`, - built: "v0.1.15-0.20261101000000-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.15-0.20261101000000-aaaaaaaaaaaa"}, - {name: "a timeout that stays fails after three downloads", asked: offHead, failing: true, failsWith: "taken 3 times", downloads: 3, - answer: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`}, + // ADR 0664 §1 (issue 1748): one download; a failure of the network is a transport fault the bus gives the ask again for after a + // delay, no retry here; another failure is not one. {name: "an unknown revision is not taken again", asked: offHead, failing: true, failsWith: "on no pull request head", downloads: 1, answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`}, - {name: "a refused connection is no pin", asked: offHead, failing: true, failsWith: "connection refused", + {name: "a refused connection is no pin", asked: offHead, failing: true, failsWith: "connection refused", passing: true, answer: `{"Path":"` + sdk + `","Error":"git ls-remote -q origin in /tmp/gopath: dial tcp 10.10.0.1:443: connect: connection refused"}`}, } { t.Run(c.name, func(t *testing.T) { h, states := newPinHarness(t, pinnedGoMod, false, pullList) h.f.gomods = "go.mod\nother/go.mod\n" - var paused []time.Duration - h.m.Pause = func(d time.Duration) { paused = append(paused, d) } downloads := 0 base := h.m.Run h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { @@ -103,9 +97,6 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) h.mu.Unlock() downloads++ - if downloads <= c.timeouts { - return forgekey.Ran{Status: 1, Stdout: `{"Path":"` + sdk + `","Error":"lookup git.novox.be on 10.10.0.3:53: i/o timeout"}`} - } if c.failing { return forgekey.Ran{Status: 1, Stdout: c.answer} } @@ -118,18 +109,18 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { t.Fatal(err) } st := states.waitEnd(t) - if c.downloads != 0 && (downloads != c.downloads || len(paused) != c.downloads-1) { - t.Errorf("go downloaded %d times with %d waits, want %d downloads", downloads, len(paused), c.downloads) + if c.downloads != 0 && downloads != c.downloads { + t.Errorf("go downloaded %d times, want %d: no retry here", downloads, c.downloads) } if c.failsWith == "on no pull request head" { // not a network failure: the forge's heads said, no retaking - if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) { + if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) || st.Passing { t.Errorf("the unknown revision ended %s: %q", st.State, st.Reason) } return } if c.failsWith != "" { if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, c.failsWith) || strings.Contains(st.Reason, "no pull request head") || - !strings.Contains(st.Reason, "may be retaken") || builds.FailedUntilChanged(st.Reason) { + strings.Contains(st.Reason, "may be retaken") || st.Passing != c.passing || builds.FailedUntilChanged(st.Reason) { t.Errorf("the download failure ended %s: %q", st.State, st.Reason) } return