C4: the conformance kit gains Passing with the words of each runner when the network did not answer; each build module's tests call it and checks/failurekit refuses one that does not (ADR 0664, issue 1748)
Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -4,7 +4,8 @@
|
||||
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
|
||||
// exempt from that, and said so. Every build module, test step or not, must also hold a test that calls
|
||||
// conformance.LeftRunning with its own start sweep, so a step a previous process left running is ended at its start
|
||||
// (issue 1890).
|
||||
// (issue 1890), and one that calls conformance.Passing, so a step whose network did not answer is said as a transport fault
|
||||
// (issue 1748).
|
||||
package failurekit
|
||||
|
||||
import (
|
||||
@@ -83,6 +84,16 @@ func Judge(root string) ([]Judged, error) {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
|
||||
"(mesh-build-go buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
}
|
||||
// Every build module meets the network (a fetch, a download, an install, a publish), so every one runs the passing case: its
|
||||
// steps say a network that did not answer as a transport fault the bus gives the ask again for (ADR 0664, issue 1748).
|
||||
says, err := runsTheKit(dir, "conformance.Passing(")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !says {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.Passing "+
|
||||
"(mesh-build-go buildstep/conformance): run the kit's passing case with the module's own path from a failed run to its step's error (issue 1748)", seat))
|
||||
}
|
||||
j.Refused = strings.Join(refused, "; ")
|
||||
out = append(out, j)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance\"\n\n"
|
||||
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n"
|
||||
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\nfunc TestPassing(t *testing.T) { conformance.Passing(t, nil, nil) }\n"
|
||||
kit := imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n" + sweep
|
||||
for path, body := range map[string]string{
|
||||
"modules/a/module.json": `{"claims":[{"name":"node-build-a"}],"tools":["build_a_fetch","build_a_test"]}`,
|
||||
@@ -25,6 +25,8 @@ func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
"modules/g/module.json": `{"claims":[{"name":"node-build-g"}],"tools":["build_g_test"]}`,
|
||||
"modules/g/cmd/g/kit_test.go": imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n",
|
||||
"modules/d/module.json": `{"claims":[{"name":"other"}],"tools":["d_test"]}`,
|
||||
"modules/h/module.json": `{"claims":[{"name":"node-build-h"}],"tools":["build_h_build"]}`, // the sweep, and no passing case
|
||||
"modules/h/cmd/h/kit_test.go": imp + "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n",
|
||||
"modules/e/module.json": `{"claims":[{"name":"node-build-e"}],"tools":["build_e_test"]}`,
|
||||
"modules/e/cmd/e/kit.go": kit, // not a test file
|
||||
} {
|
||||
@@ -40,9 +42,9 @@ func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"a": "refused", "b": "passes", "c": "exempt", "e": "refused", "f": "refused", "g": "refused"}
|
||||
want := map[string]string{"a": "refused", "b": "passes", "c": "exempt", "e": "refused", "f": "refused", "g": "refused", "h": "refused"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("judged %+v, want a, b, c, e, f and g", got)
|
||||
t.Fatalf("judged %+v, want a, b, c, e, f, g and h", got)
|
||||
}
|
||||
for _, j := range got {
|
||||
is := "passes"
|
||||
|
||||
@@ -71,3 +71,11 @@ the command is mesh-build-node's type-check line without a tsconfig.json:
|
||||
node <typescript>/bin/tsc --noEmit --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . ok.ts seat.ts
|
||||
|
||||
and the file is its stdout followed by its stderr.
|
||||
|
||||
## passing/ — a runner's words when the network did not answer (novox/hq issue 1748, ADR 0664 §1)
|
||||
|
||||
Not captured by running: a network cannot be cut on demand in a test. Each file holds the words a runner wrote when a
|
||||
build step met the fault on the mesh (the journal of ace on 2026-11-09 for ssh and go, the forge's sshd penalty; npm's
|
||||
error codes and buildah's pull failure as those programs print them), and for each a negative case, a failure of the
|
||||
source whose words are never a network's (an unknown revision, a checksum, a missing package, a failed RUN). Name:
|
||||
`<runner>-<case>.txt`; the kit's table (conformance.go, PassingCases) says which are transport faults.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Error: building at STEP "RUN false": exit status 1
|
||||
@@ -0,0 +1 @@
|
||||
Error: creating build container: initializing source docker://alpine: pinging container registry registry-1.docker.io: Get "https://registry-1.docker.io/v2/": dial tcp: lookup registry-1.docker.io on 10.10.0.3:53: i/o timeout
|
||||
@@ -0,0 +1,2 @@
|
||||
Connection closed by 10.10.0.1 port 222
|
||||
fatal: Could not read from remote repository.
|
||||
@@ -0,0 +1,6 @@
|
||||
kex_exchange_identification: read: Connection reset by peer
|
||||
Connection reset by 10.10.0.1 port 222
|
||||
fatal: Could not read from remote repository.
|
||||
|
||||
Please make sure you have the correct access rights
|
||||
and the repository exists.
|
||||
@@ -0,0 +1 @@
|
||||
fatal: couldn't find remote ref 0123456789abcdef0123456789abcdef01234567
|
||||
@@ -0,0 +1,2 @@
|
||||
ssh: Could not resolve hostname mesh-forge: Temporary failure in name resolution
|
||||
fatal: Could not read from remote repository.
|
||||
@@ -0,0 +1 @@
|
||||
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"verifying module: checksum mismatch"}
|
||||
@@ -0,0 +1 @@
|
||||
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"git.novox.be/novox/mesh-sdk/go@abc: dial tcp 10.10.0.1:5100: i/o timeout"}
|
||||
@@ -0,0 +1 @@
|
||||
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"git.novox.be/novox/mesh-sdk/go@abc: dial tcp: lookup git.novox.be on 10.10.0.3:53: read udp 10.10.0.3:42292->10.10.0.3:53: i/o timeout"}
|
||||
@@ -0,0 +1 @@
|
||||
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"invalid version: unknown revision abc"}
|
||||
@@ -0,0 +1,2 @@
|
||||
npm error code E404
|
||||
npm error 404 Not Found - GET https://git.novox.be/api/packages/novox/npm/@novox%2fx
|
||||
@@ -0,0 +1,3 @@
|
||||
npm error code EAI_AGAIN
|
||||
npm error syscall getaddrinfo
|
||||
npm error errno EAI_AGAIN
|
||||
@@ -0,0 +1,4 @@
|
||||
npm error code ECONNRESET
|
||||
npm error syscall read
|
||||
npm error errno -104
|
||||
npm error network aborted
|
||||
@@ -12,6 +12,7 @@ package conformance
|
||||
import (
|
||||
"embed"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -199,3 +200,80 @@ func describe(failed []buildstep.Failure) string {
|
||||
}
|
||||
|
||||
func quote(s string) string { return fmt.Sprintf("%q", s) }
|
||||
|
||||
// The runners whose network failures the kit holds words of (Passing).
|
||||
const (
|
||||
GitFetch = "git fetch"
|
||||
GoDownload = "go mod download"
|
||||
NpmCI = "npm ci"
|
||||
BuildahBud = "buildah bud"
|
||||
)
|
||||
|
||||
// PassingCase is the words of one runner and whether they are a transport fault: a network that did not answer, which the
|
||||
// bus gives the ask again for (novox/hq ADR 0664 §1), or a failure of the source, which fails the step at once.
|
||||
type PassingCase struct {
|
||||
Name, Runner string
|
||||
Transport bool
|
||||
}
|
||||
|
||||
// PassingCases are the kit's cases of the network's failures (captured/passing).
|
||||
var PassingCases = []PassingCase{
|
||||
{"git-fetch-kex", GitFetch, true}, {"git-fetch-closed", GitFetch, true}, {"git-fetch-resolve", GitFetch, true}, {"git-fetch-noref", GitFetch, false},
|
||||
{"go-download-lookup", GoDownload, true}, {"go-download-dial", GoDownload, true}, {"go-download-unknown", GoDownload, false}, {"go-download-checksum", GoDownload, false},
|
||||
{"npm-ci-econnreset", NpmCI, true}, {"npm-ci-eaiagain", NpmCI, true}, {"npm-ci-404", NpmCI, false},
|
||||
{"buildah-bud-timeout", BuildahBud, true}, {"buildah-bud-run", BuildahBud, false},
|
||||
}
|
||||
|
||||
// Failed is a build module's own path from a runner's words to the error its step ends with: the same code its step runs on
|
||||
// a failed command, fed the words of a runner's failed run.
|
||||
type Failed func(runner, words string) error
|
||||
|
||||
// Passing runs every case of the runners through failed, each as a subtest, and fails a build module whose step does not
|
||||
// say a network's failure as a buildstep.Passing (the bus then gives the ask again) or says a failure of the source as one.
|
||||
// A build module names the runners its steps use; the case's words are in captured/passing.
|
||||
func Passing(t *testing.T, runners []string, failed Failed) {
|
||||
t.Helper()
|
||||
ran := 0
|
||||
for _, c := range PassingCases {
|
||||
if !slices.Contains(runners, c.Runner) {
|
||||
continue
|
||||
}
|
||||
ran++
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
if problem := judgePassing(c, failed); problem != "" {
|
||||
t.Error(problem)
|
||||
}
|
||||
})
|
||||
}
|
||||
if ran == 0 {
|
||||
t.Fatalf("the kit holds no passing case of the runners %v", runners)
|
||||
}
|
||||
}
|
||||
|
||||
// JudgePassing is the problems of failed over every case of the runners: what Passing fails a build module for.
|
||||
func JudgePassing(runners []string, failed Failed) []string {
|
||||
var out []string
|
||||
for _, c := range PassingCases {
|
||||
if slices.Contains(runners, c.Runner) {
|
||||
if p := judgePassing(c, failed); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func judgePassing(c PassingCase, failed Failed) string {
|
||||
words, err := captured.ReadFile("captured/passing/" + c.Name + ".txt")
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
got := failed(c.Runner, string(words))
|
||||
switch {
|
||||
case got == nil:
|
||||
return c.Name + ": the step passed on a failed run"
|
||||
case buildstep.IsPassing(got) != c.Transport:
|
||||
return fmt.Sprintf("%s: the step ends %q, a transport fault %v, want %v", c.Name, got, buildstep.IsPassing(got), c.Transport)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package conformance
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -23,6 +24,19 @@ func TestTheFailureContract(t *testing.T) {
|
||||
})
|
||||
})
|
||||
|
||||
// ADR 0664 §1 (issue 1748): the shared wrapper keeps the passing cases; a step that never says a network's failure as a
|
||||
// transport fault, or says every failure as one, is refused.
|
||||
t.Run("the passing cases", func(t *testing.T) {
|
||||
all := []string{GitFetch, GoDownload, NpmCI, BuildahBud}
|
||||
Passing(t, all, func(_, words string) error { return buildstep.Transport(errors.New(words)) })
|
||||
if p := JudgePassing(all, func(_, words string) error { return errors.New(words) }); len(p) != 8 {
|
||||
t.Errorf("a step that wraps nothing: %d problems, want 8 (one per transport case):\n%s", len(p), strings.Join(p, "\n"))
|
||||
}
|
||||
if p := JudgePassing(all, func(_, words string) error { return &sdkstep.Passing{Err: errors.New(words)} }); len(p) != 5 {
|
||||
t.Errorf("a step that wraps everything: %d problems, want 5 (one per source failure):\n%s", len(p), strings.Join(p, "\n"))
|
||||
}
|
||||
})
|
||||
|
||||
// 1890/AC7: the SDK's start sweep keeps the left-running case, and the kit refuses a sweep that ends nothing or ends
|
||||
// everything.
|
||||
t.Run("the left-running case", func(t *testing.T) {
|
||||
|
||||
@@ -2,6 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -32,3 +36,23 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
|
||||
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
|
||||
})
|
||||
}
|
||||
|
||||
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or download to its step's error say a network that did
|
||||
// not answer as a transport fault, and a failure of the source (an unknown revision, a checksum) as none.
|
||||
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
|
||||
conformance.Passing(t, []string{conformance.GitFetch, conformance.GoDownload}, func(runner, words string) error {
|
||||
switch runner {
|
||||
case conformance.GitFetch:
|
||||
run := func(context.Context, []string, string, ...string) forgekey.Ran {
|
||||
return forgekey.Ran{Status: 128, Stderr: words}
|
||||
}
|
||||
return buildstep.FetchCommit(context.Background(), run, t.TempDir(), "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), "")
|
||||
default:
|
||||
var info downloadInfo
|
||||
if err := json.Unmarshal([]byte(words), &info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return downloadFailure(info.Error, "downloading x@y")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,6 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -17,3 +22,25 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
|
||||
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
|
||||
})
|
||||
}
|
||||
|
||||
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or image build to its step's error say a network that
|
||||
// did not answer as a transport fault, and a failure of the source (a RUN line that fails) as none.
|
||||
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
|
||||
conformance.Passing(t, []string{conformance.GitFetch, conformance.BuildahBud}, func(runner, words string) error {
|
||||
switch runner {
|
||||
case conformance.GitFetch:
|
||||
run := func(context.Context, []string, string, ...string) forgekey.Ran {
|
||||
return forgekey.Ran{Status: 128, Stderr: words}
|
||||
}
|
||||
return buildstep.FetchCommit(context.Background(), run, t.TempDir(), "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), "")
|
||||
default:
|
||||
m, f, states := setup(t, same)
|
||||
f.failBud = forgekey.Ran{Status: 125, Stderr: words}
|
||||
_, st := runBuild(t, m, states, nil)
|
||||
if st.Passing {
|
||||
return &sdkstep.Passing{Err: errors.New(st.Reason)}
|
||||
}
|
||||
return errors.New(st.Reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -30,3 +32,20 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
|
||||
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
|
||||
})
|
||||
}
|
||||
|
||||
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or install to its step's error say a network that did
|
||||
// not answer as a transport fault, and a failure of the source (a package the registry lacks) as none.
|
||||
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
|
||||
conformance.Passing(t, []string{conformance.GitFetch, conformance.NpmCI}, func(runner, words string) error {
|
||||
switch runner {
|
||||
case conformance.GitFetch:
|
||||
run := func(context.Context, []string, string, ...string) forgekey.Ran {
|
||||
return forgekey.Ran{Status: 128, Stderr: words}
|
||||
}
|
||||
return buildstep.FetchCommit(context.Background(), run, t.TempDir(), "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), "")
|
||||
default:
|
||||
bad := &fakeNpm{ran: forgekey.Ran{Status: 1, Stderr: words}}
|
||||
return toolchainIn(t, bad).Ensure(context.Background(), Recipe{Steps: "k", Lock: "k"}.Version())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user