From 5e3ae7b835b2f19bd6079f5a8f9ebf8c6058e16a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 7 Oct 2026 02:28:50 +0200 Subject: [PATCH] Replay the crash loop: a container that exits at start fails its gate on the first machine (hq ADR 0240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent server crash-looped about a hundred times behind every passing check. R-crashloop raises a container whose program exits at start, has the node-engine at its commit judge it through the runtime, and the controller at its commit judge the gate from what the engine said — two repositories, each at its own commit before the fix and on it. --- replays/cmd/prove/main.go | 40 +++++++++-- replays/crashloop_test.go | 135 ++++++++++++++++++++++++++++++++++++++ replays/docker.go | 20 +++++- replays/register.go | 27 ++++++++ replays/register_test.go | 7 +- 5 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 replays/crashloop_test.go diff --git a/replays/cmd/prove/main.go b/replays/cmd/prove/main.go index 18223eb..bc5bc86 100644 --- a/replays/cmd/prove/main.go +++ b/replays/cmd/prove/main.go @@ -42,8 +42,8 @@ func main() { if r.Before != "" { at = r.Before } - before, beforeSaid := run(r, *repos, at) - after, afterSaid := run(r, *repos, r.Fix) + before, beforeSaid := run(r, *repos, at, false) + after, afterSaid := run(r, *repos, r.Fix, true) verdict := "PROVED" if before != "fails" && before != "absent" || after != "passes" { verdict = "NOT PROVED" @@ -62,7 +62,7 @@ func main() { } // run is the replay at one commit: passes, fails, or absent (the check it replays did not exist). -func run(r replays.Replay, repos, at string) (string, string) { +func run(r replays.Replay, repos, at string, fixed bool) (string, string) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() repo := filepath.Join(repos, r.Repository) @@ -77,7 +77,7 @@ func run(r replays.Replay, repos, at string) (string, string) { defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run() switch r.Kind { - case replays.InRepository, replays.Gate: + case replays.InRepository, replays.Gate, replays.Liveness: home := filepath.Join(repos, r.Repository) if r.Home != "" { home = filepath.Join(repos, r.Home) @@ -95,6 +95,38 @@ func run(r replays.Replay, repos, at string) (string, string) { return "error", err.Error() } } + if r.Kind == replays.Liveness { + // The other repositories at their own commits on the same side, and the replay here run + // against all of them. + env := []string{"MESH_REPLAY_CONTROLLER=" + tree} + for _, also := range r.With { + ref := also.Fix + if !fixed { + ref = also.Fix + "^1" + if also.Before != "" { + ref = also.Before + } + } + other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-") + if err != nil { + return "error", err.Error() + } + defer os.RemoveAll(other) + clone := filepath.Join(repos, also.Repository) + if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other, + ref).CombinedOutput(); err != nil { + return "error", string(out) + } + defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run() + if also.Repository == "mesh-host" { + env = append(env, "MESH_REPLAY_HOST="+other) + } + } + cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package) + cmd.Env = append(os.Environ(), env...) + out, err := cmd.CombinedOutput() + return verdictOf(string(out), err, false) + } mode := "-mod=vendor" if _, err := os.Stat(filepath.Join(tree, "vendor")); err != nil { mode = "-mod=mod" diff --git a/replays/crashloop_test.go b/replays/crashloop_test.go new file mode 100644 index 0000000..db79457 --- /dev/null +++ b/replays/crashloop_test.go @@ -0,0 +1,135 @@ +package replays + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// **R-crashloop — a module's container that crash-loops fails its gate on the first machine** (novox/hq +// ADR 0240, "how it is checked", rule 1; to-be 48 Phase A, done when). +// +// On the home server the agent server restarted about a hundred times in a crash loop while the mesh read +// it applied, its tools served and no condition raised; a person found it reading its log for another +// reason (issue 268, research 032 §6). The gate judged a module by what the mesh saw from outside. +// +// The replay is the whole chain, each half at its commit: +// +// 1. a container whose program exits at start is raised here as the node-engine raises a module's — the +// runtime restarting it unless stopped, labelled with its resource; +// 2. the node-engine at MESH_REPLAY_HOST judges it through the runtime, as it judges every long-running +// resource, and writes what it states (its TestReplayCrashLoopIsSaidUnhealthy) — an engine older than +// the judging has nothing to state, and its reports carry no health; +// 3. the controller at MESH_REPLAY_CONTROLLER rolls a build of the module out, hears its first machine say +// that, and judges the gate (its TestReplayCrashLoopFailsItsGateOnTheFirstMachine). +// +// The assertion is the controller's: the build fails its gate on the first machine and is put back there. +// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is +// removed after, whatever happened. +func TestReplayCrashLoop(t *testing.T) { + host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host")) + controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller")) + for _, dir := range []string{host, controller} { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir) + } + } + if os.Getenv("MESH_TEST_POSTGRES") == "" { + t.Skip("no MESH_TEST_POSTGRES: the controller's half judges its gate against a store") + } + docker, ok := DockerFromEnv() + if !ok { + t.Skip("no container runtime: the crash loop is a container") + } + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + defer cancel() + + // 1. The crash loop, as the node-engine raises a module's container. + const image = "alpine:3.20" + if err := docker.Pull(ctx, image); err != nil { + t.Fatal(err) + } + name := fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano()) + id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"sh", "-c", "echo starting; exit 3"}, Name: name, + Restart: "unless-stopped", Labels: map[string]string{"mesh-host.id": "app.server"}}) + if err != nil { + t.Fatal(err) + } + defer docker.Remove(context.Background(), id) + + // 2. The node-engine at its commit judges it, and says what it states. + statement := filepath.Join(t.TempDir(), "statement.json") + engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopIsSaidUnhealthy$", + "./internal/liveness/") + engine.Dir = host + engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=") + out, err := engine.CombinedOutput() + switch { + case err == nil && !strings.Contains(string(out), "no tests to run"): + t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement))) + case judgesNothing(string(out)): + // An engine older than the judging: its reports carry no health. + t.Logf("the node-engine at this commit judges nothing it runs") + if err := os.WriteFile(statement, []byte("null"), 0o644); err != nil { + t.Fatal(err) + } + default: + t.Fatalf("the node-engine did not say the crash loop unhealthy: %v\n%s", err, lastOf(string(out), 12)) + } + + // 3. The controller at its commit judges the gate from what the engine said. + gate := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopFailsItsGateOnTheFirstMachine$", + "./cmd/mesh-controller/") + gate.Dir = controller + gate.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off") + out, err = gate.CombinedOutput() + if err != nil { + t.Fatalf("the gate did not fail the crash loop on its first machine: %v\n%s", err, lastOf(string(out), 12)) + } + if strings.Contains(string(out), "no tests to run") { + t.Fatal("the controller at this commit has no half of the replay to run") + } +} + +// judgesNothing is go test's word for a package that is not there: an engine older than the judging. +func judgesNothing(out string) bool { + return strings.Contains(out, "no tests to run") || strings.Contains(out, "directory not found") || + strings.Contains(out, "no Go files") || strings.Contains(out, "cannot find package") || + strings.Contains(out, "matched no packages") || strings.Contains(out, "is not in std") +} + +func statementOf(path string) string { + raw, err := os.ReadFile(path) + if err != nil { + return "(nothing written: " + err.Error() + ")" + } + return string(raw) +} + +func orDefault(s, fallback string) string { + if s == "" { + return fallback + } + return s +} + +func firstLineOf(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + if len(line) > 400 { + line = line[:400] + "…" + } + return line +} + +func lastOf(s string, n int) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} diff --git a/replays/docker.go b/replays/docker.go index bd3101e..beddac6 100644 --- a/replays/docker.go +++ b/replays/docker.go @@ -101,6 +101,11 @@ type Run struct { Binds []string Network string DNS []string + // Name, Labels and Restart are the container's name, its labels beside the replay's own, and the + // runtime's restart policy — what the node-engine creates a module's container with. + Name string + Labels map[string]string + Restart string } // Start creates and starts a container and answers its id and its address on its network. @@ -110,8 +115,19 @@ func (d *Docker) Start(ctx context.Context, r Run) (string, string, error) { if len(r.DNS) > 0 { host["Dns"] = r.DNS } - if err := d.call(ctx, http.MethodPost, "/containers/create", map[string]any{"Image": r.Image, "Cmd": r.Cmd, - "Labels": map[string]string{d.Label: "1"}, "HostConfig": host}, &made); err != nil { + if r.Restart != "" { + host["RestartPolicy"] = map[string]any{"Name": r.Restart} + } + labels := map[string]string{d.Label: "1"} + for k, v := range r.Labels { + labels[k] = v + } + path := "/containers/create" + if r.Name != "" { + path += "?name=" + url.QueryEscape(r.Name) + } + if err := d.call(ctx, http.MethodPost, path, map[string]any{"Image": r.Image, "Cmd": r.Cmd, + "Labels": labels, "HostConfig": host}, &made); err != nil { return "", "", err } if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil { diff --git a/replays/register.go b/replays/register.go index 893537a..99baafc 100644 --- a/replays/register.go +++ b/replays/register.go @@ -28,8 +28,22 @@ const ( // Gate is a test of the merge gate itself: before its fix there was no check to fail, so the commit // before is held to fail by not having it. Gate Kind = "gate" + // Liveness is the crash-loop replay here: a container that exits at start, raised on the runtime here, + // judged by the node-engine at the commit, and the controller at the commit judging its gate from what + // the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With. + Liveness Kind = "liveness" ) +// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on +// it, as Repository is. +type Also struct { + Repository string + Fix string + // Before is the commit the replay runs this repository at before the fix, when that is not Fix's + // first parent. + Before string +} + // Replay is one incident, replayed. type Replay struct { ID string @@ -51,6 +65,8 @@ type Replay struct { Files []string // Home is where the replay's files live, when that is not the fix: the repository and ref. Home, HomeRef string + // With is the other repositories the replay runs at their commits beside Repository's (Liveness). + With []Also } // Register is every replay, by incident. @@ -78,6 +94,17 @@ var Register = []Replay{ Asserts: "a consumer beside its store stays bound to it while another machine holds the store's seat", Package: "./cmd/mesh-controller", Test: "TestReplay273", Files: []string{"cmd/mesh-controller/replays_test.go"}, Home: "mesh-controller", HomeRef: "feat/replays"}, + // Not a core incident, and the first of its kind: a module's. The crash loop was found by a person + // reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR + // 0240 makes the node-engine judge it and the gate fail it. + {ID: "R-crashloop", Issue: 268, Kind: Liveness, Repository: "mesh-controller", + Fix: "feat/a-module-says-how-it-is-healthy", Before: "origin/main", + With: []Also{{Repository: "mesh-host", Fix: "feat/a-module-says-how-it-is-healthy", Before: "origin/main"}}, + What: "the agent server crash-looped about a hundred times while the mesh read it applied, its tools served " + + "and nothing raised: the gate judged a module by what the mesh saw from outside", + Asserts: "a container whose program exits at start is said unhealthy by the node-engine, and its build fails " + + "its gate on the first machine and is put back there", + Package: ".", Test: "TestReplayCrashLoop", Files: []string{"cmd/mesh-controller/replays_test.go"}}, } // Find is the replay of that id, or of that issue. diff --git a/replays/register_test.go b/replays/register_test.go index 22cdbe9..393af25 100644 --- a/replays/register_test.go +++ b/replays/register_test.go @@ -16,7 +16,8 @@ func TestEveryReplayIsWhole(t *testing.T) { if r.Issue == 0 || r.What == "" || r.Asserts == "" || r.Repository == "" || r.Fix == "" || r.Test == "" { t.Errorf("%s does not say its incident, its outcome, its fix and its test: %+v", r.ID, r) } - if !regexp.MustCompile(`^R\d+$`).MatchString(r.ID) { + // By its issue, or — for a replay that is not one core incident's — by a name of its own. + if !regexp.MustCompile(`^R(\d+|-[a-z][a-z0-9-]*)$`).MatchString(r.ID) { t.Errorf("%q is not a replay's id", r.ID) } switch r.Kind { @@ -24,6 +25,10 @@ func TestEveryReplayIsWhole(t *testing.T) { if len(r.Files) == 0 || r.Package == "" { t.Errorf("%s runs in its repository and lays no files over the commit", r.ID) } + case Liveness: + if len(r.Files) == 0 || len(r.With) == 0 { + t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r) + } case Bus, Resolver: default: t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)