jochen
1fc1e74cc3
mesh/merge-gate pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/a-module-says-how-it-is-healthy delivered: every member is delivered
A container that crash-looped after its compose applied passed every check the gate had: nothing looked at what a module runs. The node-engine now judges every long-running resource on every look — one read of the runtime, one per service manager — keeps the restarts it counts across recreates and its own restarts, and says the state in every report and as an event on change, again every minute while not healthy. It reads only; nothing is restarted for being unhealthy.
92 lines
3.6 KiB
Go
92 lines
3.6 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
// **The crash loop, on a real runtime** — the engine's half of mesh-lab's replay R-crashloop (novox/hq
|
|
// ADR 0240, "how it is checked", rule 1: "a mesh-lab replay of the crash loop (a container whose program
|
|
// exits at start) fails its gate within the bound").
|
|
//
|
|
// A container whose program exits at once, started as the apply starts every container — detached,
|
|
// restarted by the runtime unless stopped, labelled with its resource — is judged by this engine through
|
|
// the runtime's own command line, and said unhealthy, restarting, inside the gate's bound. What it said is
|
|
// written to MESH_REPLAY_STATEMENT, for the controller's half to judge its gate from.
|
|
//
|
|
// Run by the lab (`go run ./replays/cmd/prove R-crashloop`), which starts the container and names it in
|
|
// MESH_REPLAY_CONTAINER; or by hand with MESH_REPLAY_RUNTIME=1, starting its own. Skipped otherwise: the
|
|
// suite raises no container unasked. The grace is shortened to seconds, which changes the clock and not
|
|
// the rule.
|
|
func TestReplayCrashLoopIsSaidUnhealthy(t *testing.T) {
|
|
name := os.Getenv("MESH_REPLAY_CONTAINER")
|
|
if name == "" && os.Getenv("MESH_REPLAY_RUNTIME") != "1" {
|
|
t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container")
|
|
}
|
|
run := func(ctx context.Context, cmd string, args ...string) (string, error) {
|
|
c := exec.CommandContext(ctx, cmd, args...)
|
|
out, err := c.Output()
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) {
|
|
return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
|
}
|
|
return string(out), err
|
|
}
|
|
if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
|
|
t.Skipf("no container runtime answers here: %v", err)
|
|
}
|
|
if name == "" {
|
|
name = fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
|
|
if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped",
|
|
"--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", "alpine:3.20", "sh", "-c", "exit 3"); err != nil {
|
|
t.Fatalf("starting the crash loop: %v %s", err, out)
|
|
}
|
|
t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) })
|
|
}
|
|
|
|
j, err := Open("", &Exec{Run: run})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Grace = 3 * time.Second
|
|
j.Set([]Resource{{Module: "app", ID: "app.server", Kind: KindContainer, Target: name}})
|
|
began := time.Now()
|
|
deadline := began.Add(2 * time.Minute)
|
|
var st Statement
|
|
for time.Now().Before(deadline) {
|
|
st, _ = j.Look(t.Context())
|
|
if r := st.Resources[0]; r.State == Unhealthy && r.Restarts >= 2 {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
s := st.Resources[0]
|
|
if s.State != Unhealthy || s.Restarts < 2 {
|
|
t.Fatalf("a container whose program exits at start was not said unhealthy, with its restarts counted, in two minutes: %+v", s)
|
|
}
|
|
t.Logf("said %s (%s), %d restarts counted, %s after the judging began", s.State, s.Reason, s.Restarts,
|
|
time.Since(began).Round(time.Second))
|
|
|
|
if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" {
|
|
h := link.Health{Contract: link.LivenessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{
|
|
Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason,
|
|
Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts}}}
|
|
raw, err := json.Marshal(h)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(out, raw, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|