Keep what each machine says of what it runs, raise it, and gate on it (hq ADR 0240, to-be 48 Phase A)
The gate judged a module by what the mesh saw from outside, so a container that crash-looped after it applied passed it. Each machine's node-engine now states the health of every long-running resource it runs; the controller keeps the newest statement per machine, raises module.<module>.<machine>.unhealthy on the second statement in a row, clears it on the first that does not say it, and the gate passes a module only when every long-running resource of it is stated healthy since the send. An engine that states nothing is judged as before.
This commit is contained in:
jochen
@@ -1,9 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
||||
@@ -111,3 +119,170 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
||||
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
||||
}
|
||||
}
|
||||
|
||||
// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq
|
||||
// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the
|
||||
// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the
|
||||
// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container
|
||||
// exits at start never passes its gate on the first machine, is put back there at the bound, and never
|
||||
// reaches the second.
|
||||
//
|
||||
// The machine is heard as a node-engine says it: its report of the apply, then the same account said again
|
||||
// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the
|
||||
// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the
|
||||
// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of
|
||||
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
||||
`"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` +
|
||||
`"streak":5,"restarts":2}]}`)
|
||||
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("the engine's statement of the crash loop: %v", err)
|
||||
}
|
||||
crashLoop = raw
|
||||
}
|
||||
// `null` is an engine that states nothing — older than the judging — and its reports carry no health.
|
||||
var stated map[string]any
|
||||
if err := json.Unmarshal(crashLoop, &stated); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
registerAt := func(commit string, asked time.Time) {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
||||
Head: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
registerAt("c1", time.Now().Add(-2*time.Hour))
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
registerAt("c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// The machine: what it is sent is applied, and its report says what runs is starting.
|
||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||
sequence := int64(0)
|
||||
say := func(node, digest, state string) {
|
||||
t.Helper()
|
||||
sequence++
|
||||
health := map[string]any{}
|
||||
for k, v := range stated {
|
||||
health[k] = v
|
||||
}
|
||||
health["at"] = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
if state != "" && stated != nil {
|
||||
resources := []any{}
|
||||
for _, r := range stated["resources"].([]any) {
|
||||
kept := map[string]any{}
|
||||
for k, v := range r.(map[string]any) {
|
||||
kept[k] = v
|
||||
}
|
||||
kept["state"], kept["reason"] = state, ""
|
||||
resources = append(resources, kept)
|
||||
}
|
||||
health["resources"] = resources
|
||||
}
|
||||
said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest,
|
||||
"report_sequence": sequence}
|
||||
if stated != nil {
|
||||
said["health"] = health
|
||||
}
|
||||
body, _ := json.Marshal(said)
|
||||
var report link.Report
|
||||
if err := json.Unmarshal(body, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listener.Heard(ctx, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var sent [][]string
|
||||
last := map[string]string{}
|
||||
n := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
last[node] = digest
|
||||
say(node, digest, "starting")
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery = 0, 0
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, open) // the first machine is sent the new build, and starts it
|
||||
if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) {
|
||||
t.Fatalf("sent %v, not the first machine first", sent)
|
||||
}
|
||||
advancePlans(ctx, open) // judged while it starts
|
||||
// Its container exits at start, and the runtime restarts it: the machine says so, again and again.
|
||||
for i := 0; i < 6 && len(sent) == 1; i++ {
|
||||
say("anchor", last["anchor"], "")
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
gateBound = -time.Second // and the bound passes
|
||||
advancePlans(ctx, open)
|
||||
|
||||
p, err := inv.PlanByID(ctx, "plan-crashloop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := p.Modules["app"].Gate
|
||||
for _, names := range sent {
|
||||
if slices.Contains(names, "laptop") {
|
||||
t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate)
|
||||
}
|
||||
}
|
||||
if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed {
|
||||
t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v",
|
||||
p.State, p.Note, gate)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user