Replay the crash loop: a container that exits at start fails its gate on the first machine (hq ADR 0240)

The agent server crash-looped about a hundred times behind every passing check.
R-crashloop raises a container whose program exits at start, has the node-engine
at its commit judge it through the runtime, and the controller at its commit
judge the gate from what the engine said — two repositories, each at its own
commit before the fix and on it.

replays/cmd/prove/main.go

@@ -42,8 +42,8 @@ func main() {
 		if r.Before != "" {
 			at = r.Before
 		}
-		before, beforeSaid := run(r, *repos, at)
+		before, beforeSaid := run(r, *repos, at, false)
-		after, afterSaid := run(r, *repos, r.Fix)
+		after, afterSaid := run(r, *repos, r.Fix, true)
 		verdict := "PROVED"
 		if before != "fails" && before != "absent" || after != "passes" {
 			verdict = "NOT PROVED"
@@ -62,7 +62,7 @@ func main() {
 }
 
 // run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
-func run(r replays.Replay, repos, at string) (string, string) {
+func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
 	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 	defer cancel()
 	repo := filepath.Join(repos, r.Repository)
@@ -77,7 +77,7 @@ func run(r replays.Replay, repos, at string) (string, string) {
 	defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
 
 	switch r.Kind {
-	case replays.InRepository, replays.Gate:
+	case replays.InRepository, replays.Gate, replays.Liveness:
 		home := filepath.Join(repos, r.Repository)
 		if r.Home != "" {
 			home = filepath.Join(repos, r.Home)
@@ -95,6 +95,38 @@ func run(r replays.Replay, repos, at string) (string, string) {
 				return "error", err.Error()
 			}
 		}
+		if r.Kind == replays.Liveness {
+			// The other repositories at their own commits on the same side, and the replay here run
+			// against all of them.
+			env := []string{"MESH_REPLAY_CONTROLLER=" + tree}
+			for _, also := range r.With {
+				ref := also.Fix
+				if !fixed {
+					ref = also.Fix + "^1"
+					if also.Before != "" {
+						ref = also.Before
+					}
+				}
+				other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
+				if err != nil {
+					return "error", err.Error()
+				}
+				defer os.RemoveAll(other)
+				clone := filepath.Join(repos, also.Repository)
+				if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
+					ref).CombinedOutput(); err != nil {
+					return "error", string(out)
+				}
+				defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
+				if also.Repository == "mesh-host" {
+					env = append(env, "MESH_REPLAY_HOST="+other)
+				}
+			}
+			cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package)
+			cmd.Env = append(os.Environ(), env...)
+			out, err := cmd.CombinedOutput()
+			return verdictOf(string(out), err, false)
+		}
 		mode := "-mod=vendor"
 		if _, err := os.Stat(filepath.Join(tree, "vendor")); err != nil {
 			mode = "-mod=mod"

replays/crashloop_test.go

@@ -0,0 +1,135 @@
+package replays
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// **R-crashloop — a module's container that crash-loops fails its gate on the first machine** (novox/hq
+// ADR 0240, "how it is checked", rule 1; to-be 48 Phase A, done when).
+//
+// On the home server the agent server restarted about a hundred times in a crash loop while the mesh read
+// it applied, its tools served and no condition raised; a person found it reading its log for another
+// reason (issue 268, research 032 §6). The gate judged a module by what the mesh saw from outside.
+//
+// The replay is the whole chain, each half at its commit:
+//
+//  1. a container whose program exits at start is raised here as the node-engine raises a module's — the
+//     runtime restarting it unless stopped, labelled with its resource;
+//  2. the node-engine at MESH_REPLAY_HOST judges it through the runtime, as it judges every long-running
+//     resource, and writes what it states (its TestReplayCrashLoopIsSaidUnhealthy) — an engine older than
+//     the judging has nothing to state, and its reports carry no health;
+//  3. the controller at MESH_REPLAY_CONTROLLER rolls a build of the module out, hears its first machine say
+//     that, and judges the gate (its TestReplayCrashLoopFailsItsGateOnTheFirstMachine).
+//
+// The assertion is the controller's: the build fails its gate on the first machine and is put back there.
+// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is
+// removed after, whatever happened.
+func TestReplayCrashLoop(t *testing.T) {
+	host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host"))
+	controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller"))
+	for _, dir := range []string{host, controller} {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
+			t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir)
+		}
+	}
+	if os.Getenv("MESH_TEST_POSTGRES") == "" {
+		t.Skip("no MESH_TEST_POSTGRES: the controller's half judges its gate against a store")
+	}
+	docker, ok := DockerFromEnv()
+	if !ok {
+		t.Skip("no container runtime: the crash loop is a container")
+	}
+	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
+	defer cancel()
+
+	// 1. The crash loop, as the node-engine raises a module's container.
+	const image = "alpine:3.20"
+	if err := docker.Pull(ctx, image); err != nil {
+		t.Fatal(err)
+	}
+	name := fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
+	id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"sh", "-c", "echo starting; exit 3"}, Name: name,
+		Restart: "unless-stopped", Labels: map[string]string{"mesh-host.id": "app.server"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer docker.Remove(context.Background(), id)
+
+	// 2. The node-engine at its commit judges it, and says what it states.
+	statement := filepath.Join(t.TempDir(), "statement.json")
+	engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopIsSaidUnhealthy$",
+		"./internal/liveness/")
+	engine.Dir = host
+	engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=")
+	out, err := engine.CombinedOutput()
+	switch {
+	case err == nil && !strings.Contains(string(out), "no tests to run"):
+		t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement)))
+	case judgesNothing(string(out)):
+		// An engine older than the judging: its reports carry no health.
+		t.Logf("the node-engine at this commit judges nothing it runs")
+		if err := os.WriteFile(statement, []byte("null"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	default:
+		t.Fatalf("the node-engine did not say the crash loop unhealthy: %v\n%s", err, lastOf(string(out), 12))
+	}
+
+	// 3. The controller at its commit judges the gate from what the engine said.
+	gate := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopFailsItsGateOnTheFirstMachine$",
+		"./cmd/mesh-controller/")
+	gate.Dir = controller
+	gate.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off")
+	out, err = gate.CombinedOutput()
+	if err != nil {
+		t.Fatalf("the gate did not fail the crash loop on its first machine: %v\n%s", err, lastOf(string(out), 12))
+	}
+	if strings.Contains(string(out), "no tests to run") {
+		t.Fatal("the controller at this commit has no half of the replay to run")
+	}
+}
+
+// judgesNothing is go test's word for a package that is not there: an engine older than the judging.
+func judgesNothing(out string) bool {
+	return strings.Contains(out, "no tests to run") || strings.Contains(out, "directory not found") ||
+		strings.Contains(out, "no Go files") || strings.Contains(out, "cannot find package") ||
+		strings.Contains(out, "matched no packages") || strings.Contains(out, "is not in std")
+}
+
+func statementOf(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "(nothing written: " + err.Error() + ")"
+	}
+	return string(raw)
+}
+
+func orDefault(s, fallback string) string {
+	if s == "" {
+		return fallback
+	}
+	return s
+}
+
+func firstLineOf(s string) string {
+	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
+	if len(line) > 400 {
+		line = line[:400] + "…"
+	}
+	return line
+}
+
+func lastOf(s string, n int) string {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}

replays/docker.go

@@ -101,6 +101,11 @@ type Run struct {
 	Binds   []string
 	Network string
 	DNS     []string
+	// Name, Labels and Restart are the container's name, its labels beside the replay's own, and the
+	// runtime's restart policy — what the node-engine creates a module's container with.
+	Name    string
+	Labels  map[string]string
+	Restart string
 }
 
 // Start creates and starts a container and answers its id and its address on its network.
@@ -110,8 +115,19 @@ func (d *Docker) Start(ctx context.Context, r Run) (string, string, error) {
 	if len(r.DNS) > 0 {
 		host["Dns"] = r.DNS
 	}
-	if err := d.call(ctx, http.MethodPost, "/containers/create", map[string]any{"Image": r.Image, "Cmd": r.Cmd,
+	if r.Restart != "" {
-		"Labels": map[string]string{d.Label: "1"}, "HostConfig": host}, &made); err != nil {
+		host["RestartPolicy"] = map[string]any{"Name": r.Restart}
+	}
+	labels := map[string]string{d.Label: "1"}
+	for k, v := range r.Labels {
+		labels[k] = v
+	}
+	path := "/containers/create"
+	if r.Name != "" {
+		path += "?name=" + url.QueryEscape(r.Name)
+	}
+	if err := d.call(ctx, http.MethodPost, path, map[string]any{"Image": r.Image, "Cmd": r.Cmd,
+		"Labels": labels, "HostConfig": host}, &made); err != nil {
 		return "", "", err
 	}
 	if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {

replays/register.go

@@ -28,8 +28,22 @@ const (
 	// Gate is a test of the merge gate itself: before its fix there was no check to fail, so the commit
 	// before is held to fail by not having it.
 	Gate Kind = "gate"
+	// Liveness is the crash-loop replay here: a container that exits at start, raised on the runtime here,
+	// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
+	// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
+	Liveness Kind = "liveness"
 )
 
+// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
+// it, as Repository is.
+type Also struct {
+	Repository string
+	Fix        string
+	// Before is the commit the replay runs this repository at before the fix, when that is not Fix's
+	// first parent.
+	Before string
+}
+
 // Replay is one incident, replayed.
 type Replay struct {
 	ID    string
@@ -51,6 +65,8 @@ type Replay struct {
 	Files   []string
 	// Home is where the replay's files live, when that is not the fix: the repository and ref.
 	Home, HomeRef string
+	// With is the other repositories the replay runs at their commits beside Repository's (Liveness).
+	With []Also
 }
 
 // Register is every replay, by incident.
@@ -78,6 +94,17 @@ var Register = []Replay{
 		Asserts: "a consumer beside its store stays bound to it while another machine holds the store's seat",
 		Package: "./cmd/mesh-controller", Test: "TestReplay273", Files: []string{"cmd/mesh-controller/replays_test.go"},
 		Home: "mesh-controller", HomeRef: "feat/replays"},
+	// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
+	// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
+	// 0240 makes the node-engine judge it and the gate fail it.
+	{ID: "R-crashloop", Issue: 268, Kind: Liveness, Repository: "mesh-controller",
+		Fix: "feat/a-module-says-how-it-is-healthy", Before: "origin/main",
+		With: []Also{{Repository: "mesh-host", Fix: "feat/a-module-says-how-it-is-healthy", Before: "origin/main"}},
+		What: "the agent server crash-looped about a hundred times while the mesh read it applied, its tools served " +
+			"and nothing raised: the gate judged a module by what the mesh saw from outside",
+		Asserts: "a container whose program exits at start is said unhealthy by the node-engine, and its build fails " +
+			"its gate on the first machine and is put back there",
+		Package: ".", Test: "TestReplayCrashLoop", Files: []string{"cmd/mesh-controller/replays_test.go"}},
 }
 
 // Find is the replay of that id, or of that issue.

replays/register_test.go

@@ -16,7 +16,8 @@ func TestEveryReplayIsWhole(t *testing.T) {
 		if r.Issue == 0 || r.What == "" || r.Asserts == "" || r.Repository == "" || r.Fix == "" || r.Test == "" {
 			t.Errorf("%s does not say its incident, its outcome, its fix and its test: %+v", r.ID, r)
 		}
-		if !regexp.MustCompile(`^R\d+$`).MatchString(r.ID) {
+		// By its issue, or — for a replay that is not one core incident's — by a name of its own.
+		if !regexp.MustCompile(`^R(\d+|-[a-z][a-z0-9-]*)$`).MatchString(r.ID) {
 			t.Errorf("%q is not a replay's id", r.ID)
 		}
 		switch r.Kind {
@@ -24,6 +25,10 @@ func TestEveryReplayIsWhole(t *testing.T) {
 			if len(r.Files) == 0 || r.Package == "" {
 				t.Errorf("%s runs in its repository and lays no files over the commit", r.ID)
 			}
+		case Liveness:
+			if len(r.Files) == 0 || len(r.With) == 0 {
+				t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r)
+			}
 		case Bus, Resolver:
 		default:
 			t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)