C4: the conformance kit gains Passing with the words of each runner when the network did not answer; each build module's tests call it and checks/failurekit refuses one that does not (ADR 0664, issue 1748)

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
2026-11-09 23:15:35 +01:00
parent bdfd8db265
commit 06f28fbadb
21 changed files with 213 additions and 4 deletions
+12 -1
View File
@@ -4,7 +4,8 @@
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
// exempt from that, and said so. Every build module, test step or not, must also hold a test that calls
// conformance.LeftRunning with its own start sweep, so a step a previous process left running is ended at its start
// (issue 1890).
// (issue 1890), and one that calls conformance.Passing, so a step whose network did not answer is said as a transport fault
// (issue 1748).
package failurekit
import (
@@ -83,6 +84,16 @@ func Judge(root string) ([]Judged, error) {
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
"(mesh-build-go buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
}
// Every build module meets the network (a fetch, a download, an install, a publish), so every one runs the passing case: its
// steps say a network that did not answer as a transport fault the bus gives the ask again for (ADR 0664, issue 1748).
says, err := runsTheKit(dir, "conformance.Passing(")
if err != nil {
return nil, err
}
if !says {
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.Passing "+
"(mesh-build-go buildstep/conformance): run the kit's passing case with the module's own path from a failed run to its step's error (issue 1748)", seat))
}
j.Refused = strings.Join(refused, "; ")
out = append(out, j)
}
+5 -3
View File
@@ -12,7 +12,7 @@ import (
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
root := t.TempDir()
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance\"\n\n"
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n"
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\nfunc TestPassing(t *testing.T) { conformance.Passing(t, nil, nil) }\n"
kit := imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n" + sweep
for path, body := range map[string]string{
"modules/a/module.json": `{"claims":[{"name":"node-build-a"}],"tools":["build_a_fetch","build_a_test"]}`,
@@ -25,6 +25,8 @@ func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
"modules/g/module.json": `{"claims":[{"name":"node-build-g"}],"tools":["build_g_test"]}`,
"modules/g/cmd/g/kit_test.go": imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n",
"modules/d/module.json": `{"claims":[{"name":"other"}],"tools":["d_test"]}`,
"modules/h/module.json": `{"claims":[{"name":"node-build-h"}],"tools":["build_h_build"]}`, // the sweep, and no passing case
"modules/h/cmd/h/kit_test.go": imp + "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n",
"modules/e/module.json": `{"claims":[{"name":"node-build-e"}],"tools":["build_e_test"]}`,
"modules/e/cmd/e/kit.go": kit, // not a test file
} {
@@ -40,9 +42,9 @@ func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
if err != nil {
t.Fatal(err)
}
want := map[string]string{"a": "refused", "b": "passes", "c": "exempt", "e": "refused", "f": "refused", "g": "refused"}
want := map[string]string{"a": "refused", "b": "passes", "c": "exempt", "e": "refused", "f": "refused", "g": "refused", "h": "refused"}
if len(got) != len(want) {
t.Fatalf("judged %+v, want a, b, c, e, f and g", got)
t.Fatalf("judged %+v, want a, b, c, e, f, g and h", got)
}
for _, j := range got {
is := "passes"
@@ -71,3 +71,11 @@ the command is mesh-build-node's type-check line without a tsconfig.json:
node <typescript>/bin/tsc --noEmit --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . ok.ts seat.ts
and the file is its stdout followed by its stderr.
## passing/ — a runner's words when the network did not answer (novox/hq issue 1748, ADR 0664 §1)
Not captured by running: a network cannot be cut on demand in a test. Each file holds the words a runner wrote when a
build step met the fault on the mesh (the journal of ace on 2026-11-09 for ssh and go, the forge's sshd penalty; npm's
error codes and buildah's pull failure as those programs print them), and for each a negative case, a failure of the
source whose words are never a network's (an unknown revision, a checksum, a missing package, a failed RUN). Name:
`<runner>-<case>.txt`; the kit's table (conformance.go, PassingCases) says which are transport faults.
@@ -0,0 +1 @@
Error: building at STEP "RUN false": exit status 1
@@ -0,0 +1 @@
Error: creating build container: initializing source docker://alpine: pinging container registry registry-1.docker.io: Get "https://registry-1.docker.io/v2/": dial tcp: lookup registry-1.docker.io on 10.10.0.3:53: i/o timeout
@@ -0,0 +1,2 @@
Connection closed by 10.10.0.1 port 222
fatal: Could not read from remote repository.
@@ -0,0 +1,6 @@
kex_exchange_identification: read: Connection reset by peer
Connection reset by 10.10.0.1 port 222
fatal: Could not read from remote repository.
Please make sure you have the correct access rights
and the repository exists.
@@ -0,0 +1 @@
fatal: couldn't find remote ref 0123456789abcdef0123456789abcdef01234567
@@ -0,0 +1,2 @@
ssh: Could not resolve hostname mesh-forge: Temporary failure in name resolution
fatal: Could not read from remote repository.
@@ -0,0 +1 @@
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"verifying module: checksum mismatch"}
@@ -0,0 +1 @@
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"git.novox.be/novox/mesh-sdk/go@abc: dial tcp 10.10.0.1:5100: i/o timeout"}
@@ -0,0 +1 @@
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"git.novox.be/novox/mesh-sdk/go@abc: dial tcp: lookup git.novox.be on 10.10.0.3:53: read udp 10.10.0.3:42292->10.10.0.3:53: i/o timeout"}
@@ -0,0 +1 @@
{"Path":"git.novox.be/novox/mesh-sdk/go","Error":"invalid version: unknown revision abc"}
@@ -0,0 +1,2 @@
npm error code E404
npm error 404 Not Found - GET https://git.novox.be/api/packages/novox/npm/@novox%2fx
@@ -0,0 +1,3 @@
npm error code EAI_AGAIN
npm error syscall getaddrinfo
npm error errno EAI_AGAIN
@@ -0,0 +1,4 @@
npm error code ECONNRESET
npm error syscall read
npm error errno -104
npm error network aborted
@@ -12,6 +12,7 @@ package conformance
import (
"embed"
"fmt"
"slices"
"strings"
"testing"
@@ -199,3 +200,80 @@ func describe(failed []buildstep.Failure) string {
}
func quote(s string) string { return fmt.Sprintf("%q", s) }
// The runners whose network failures the kit holds words of (Passing).
const (
GitFetch = "git fetch"
GoDownload = "go mod download"
NpmCI = "npm ci"
BuildahBud = "buildah bud"
)
// PassingCase is the words of one runner and whether they are a transport fault: a network that did not answer, which the
// bus gives the ask again for (novox/hq ADR 0664 §1), or a failure of the source, which fails the step at once.
type PassingCase struct {
Name, Runner string
Transport bool
}
// PassingCases are the kit's cases of the network's failures (captured/passing).
var PassingCases = []PassingCase{
{"git-fetch-kex", GitFetch, true}, {"git-fetch-closed", GitFetch, true}, {"git-fetch-resolve", GitFetch, true}, {"git-fetch-noref", GitFetch, false},
{"go-download-lookup", GoDownload, true}, {"go-download-dial", GoDownload, true}, {"go-download-unknown", GoDownload, false}, {"go-download-checksum", GoDownload, false},
{"npm-ci-econnreset", NpmCI, true}, {"npm-ci-eaiagain", NpmCI, true}, {"npm-ci-404", NpmCI, false},
{"buildah-bud-timeout", BuildahBud, true}, {"buildah-bud-run", BuildahBud, false},
}
// Failed is a build module's own path from a runner's words to the error its step ends with: the same code its step runs on
// a failed command, fed the words of a runner's failed run.
type Failed func(runner, words string) error
// Passing runs every case of the runners through failed, each as a subtest, and fails a build module whose step does not
// say a network's failure as a buildstep.Passing (the bus then gives the ask again) or says a failure of the source as one.
// A build module names the runners its steps use; the case's words are in captured/passing.
func Passing(t *testing.T, runners []string, failed Failed) {
t.Helper()
ran := 0
for _, c := range PassingCases {
if !slices.Contains(runners, c.Runner) {
continue
}
ran++
t.Run(c.Name, func(t *testing.T) {
if problem := judgePassing(c, failed); problem != "" {
t.Error(problem)
}
})
}
if ran == 0 {
t.Fatalf("the kit holds no passing case of the runners %v", runners)
}
}
// JudgePassing is the problems of failed over every case of the runners: what Passing fails a build module for.
func JudgePassing(runners []string, failed Failed) []string {
var out []string
for _, c := range PassingCases {
if slices.Contains(runners, c.Runner) {
if p := judgePassing(c, failed); p != "" {
out = append(out, p)
}
}
}
return out
}
func judgePassing(c PassingCase, failed Failed) string {
words, err := captured.ReadFile("captured/passing/" + c.Name + ".txt")
if err != nil {
return err.Error()
}
got := failed(c.Runner, string(words))
switch {
case got == nil:
return c.Name + ": the step passed on a failed run"
case buildstep.IsPassing(got) != c.Transport:
return fmt.Sprintf("%s: the step ends %q, a transport fault %v, want %v", c.Name, got, buildstep.IsPassing(got), c.Transport)
}
return ""
}
@@ -3,6 +3,7 @@ package conformance
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
@@ -23,6 +24,19 @@ func TestTheFailureContract(t *testing.T) {
})
})
// ADR 0664 §1 (issue 1748): the shared wrapper keeps the passing cases; a step that never says a network's failure as a
// transport fault, or says every failure as one, is refused.
t.Run("the passing cases", func(t *testing.T) {
all := []string{GitFetch, GoDownload, NpmCI, BuildahBud}
Passing(t, all, func(_, words string) error { return buildstep.Transport(errors.New(words)) })
if p := JudgePassing(all, func(_, words string) error { return errors.New(words) }); len(p) != 8 {
t.Errorf("a step that wraps nothing: %d problems, want 8 (one per transport case):\n%s", len(p), strings.Join(p, "\n"))
}
if p := JudgePassing(all, func(_, words string) error { return &sdkstep.Passing{Err: errors.New(words)} }); len(p) != 5 {
t.Errorf("a step that wraps everything: %d problems, want 5 (one per source failure):\n%s", len(p), strings.Join(p, "\n"))
}
})
// 1890/AC7: the SDK's start sweep keeps the left-running case, and the kit refuses a sweep that ends nothing or ends
// everything.
t.Run("the left-running case", func(t *testing.T) {
@@ -2,6 +2,10 @@ package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"strings"
"testing"
"time"
@@ -32,3 +36,23 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
})
}
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or download to its step's error say a network that did
// not answer as a transport fault, and a failure of the source (an unknown revision, a checksum) as none.
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
conformance.Passing(t, []string{conformance.GitFetch, conformance.GoDownload}, func(runner, words string) error {
switch runner {
case conformance.GitFetch:
run := func(context.Context, []string, string, ...string) forgekey.Ran {
return forgekey.Ran{Status: 128, Stderr: words}
}
return buildstep.FetchCommit(context.Background(), run, t.TempDir(), "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), "")
default:
var info downloadInfo
if err := json.Unmarshal([]byte(words), &info); err != nil {
t.Fatal(err)
}
return downloadFailure(info.Error, "downloading x@y")
}
})
}
@@ -2,6 +2,11 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"strings"
"testing"
"time"
@@ -17,3 +22,25 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
})
}
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or image build to its step's error say a network that
// did not answer as a transport fault, and a failure of the source (a RUN line that fails) as none.
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
conformance.Passing(t, []string{conformance.GitFetch, conformance.BuildahBud}, func(runner, words string) error {
switch runner {
case conformance.GitFetch:
run := func(context.Context, []string, string, ...string) forgekey.Ran {
return forgekey.Ran{Status: 128, Stderr: words}
}
return buildstep.FetchCommit(context.Background(), run, t.TempDir(), "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), "")
default:
m, f, states := setup(t, same)
f.failBud = forgekey.Ran{Status: 125, Stderr: words}
_, st := runBuild(t, m, states, nil)
if st.Passing {
return &sdkstep.Passing{Err: errors.New(st.Reason)}
}
return errors.New(st.Reason)
}
})
}
@@ -2,6 +2,8 @@ package main
import (
"context"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"strings"
"testing"
"time"
@@ -30,3 +32,20 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
})
}
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or install to its step's error say a network that did
// not answer as a transport fault, and a failure of the source (a package the registry lacks) as none.
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
conformance.Passing(t, []string{conformance.GitFetch, conformance.NpmCI}, func(runner, words string) error {
switch runner {
case conformance.GitFetch:
run := func(context.Context, []string, string, ...string) forgekey.Ran {
return forgekey.Ran{Status: 128, Stderr: words}
}
return buildstep.FetchCommit(context.Background(), run, t.TempDir(), "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), "")
default:
bad := &fakeNpm{ran: forgekey.Ran{Status: 1, Stderr: words}}
return toolchainIn(t, bad).Ensure(context.Background(), Recipe{Steps: "k", Lock: "k"}.Version())
}
})
}