jochen
1cc6a2d759
The gate judged a module by what the mesh saw from outside, so a container that crash-looped after it applied passed it. Each machine's node-engine now states the health of every long-running resource it runs; the controller keeps the newest statement per machine, raises module.<module>.<machine>.unhealthy on the second statement in a row, clears it on the first that does not say it, and the gate passes a module only when every long-running resource of it is stated healthy since the send. An engine that states nothing is judged as before.
289 lines
12 KiB
Go
289 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
|
|
// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the
|
|
// commit before the fix and fail there, and against the fix and pass. **Written only with what the
|
|
// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the
|
|
// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there.
|
|
//
|
|
// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's
|
|
// merge check on every pull request with the rest of the suite.
|
|
|
|
// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network
|
|
// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26
|
|
// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every
|
|
// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the
|
|
// provider's machine composes whatever its consumers are called, and a provision that mints no
|
|
// credential holds no consumer to a key's length.
|
|
func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
|
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
|
Requires: []string{"wildcard-resolution"}})
|
|
for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} {
|
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
|
}
|
|
}
|
|
// The consumer's machine composes first, as a push does: what it is sent is what its provider grants.
|
|
for _, node := range []string{"laptop", "anchor"} {
|
|
plan, settings, err := planFor(ctx, open, node)
|
|
if err != nil {
|
|
t.Fatalf("%s does not resolve: %v", node, err)
|
|
}
|
|
if _, err := declarationFor(ctx, open, node, plan, settings); err != nil {
|
|
t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+
|
|
"mesh_laptop_networkmanager (26 characters), refused it: %v", node, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for
|
|
// the resolver re-bound every database consumer on the home server — which runs its own store, beside
|
|
// its applications' data — to the store seat's holder on the control node, which made each a new empty
|
|
// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a
|
|
// machine running its own store stays bound to it while another machine holds the store's seat; the
|
|
// resolver, which every holder answers alike, follows its seat.
|
|
func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, m := range []catalogue.Manifest{
|
|
{Module: "store", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
|
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
|
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
|
Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}},
|
|
{Module: "resolver", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
|
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}},
|
|
{Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
|
{Module: "board", Version: "1", Requires: []string{"postgres-database"}},
|
|
} {
|
|
register(t, open, m)
|
|
}
|
|
assignAll := func(pairs ...[2]string) {
|
|
for _, a := range pairs {
|
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
|
}
|
|
}
|
|
}
|
|
// The control node holds the mesh's store and resolver seats; the home server runs its own of each.
|
|
assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"})
|
|
for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} {
|
|
if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"},
|
|
[2]string{"laptop", "board"})
|
|
|
|
plan, _, err := planFor(ctx, open, "laptop")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var board, network string
|
|
for _, n := range plan.Needs {
|
|
switch {
|
|
case n.For == "board" && n.Name == "postgres-database":
|
|
board = n.From
|
|
case n.For == "network" && n.Name == "wildcard-resolution":
|
|
network = n.From
|
|
}
|
|
}
|
|
if board != "laptop" {
|
|
t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+
|
|
"database there (issue 273)", board)
|
|
}
|
|
if network != "anchor" {
|
|
t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
|
|
}
|
|
}
|
|
|
|
// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq
|
|
// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the
|
|
// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the
|
|
// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container
|
|
// exits at start never passes its gate on the first machine, is put back there at the bound, and never
|
|
// reaches the second.
|
|
//
|
|
// The machine is heard as a node-engine says it: its report of the apply, then the same account said again
|
|
// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the
|
|
// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the
|
|
// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of
|
|
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
|
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
|
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
|
|
crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
|
`"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` +
|
|
`"streak":5,"restarts":2}]}`)
|
|
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("the engine's statement of the crash loop: %v", err)
|
|
}
|
|
crashLoop = raw
|
|
}
|
|
// `null` is an engine that states nothing — older than the judging — and its reports carry no health.
|
|
var stated map[string]any
|
|
if err := json.Unmarshal(crashLoop, &stated); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, b := range []inventory.Build{
|
|
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
|
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
|
} {
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
|
b.Manifest = manifest
|
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registerAt := func(commit string, asked time.Time) {
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
|
Head: commit, Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registerAt("c1", time.Now().Add(-2*time.Hour))
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registerAt("c2", time.Now().Add(-time.Minute))
|
|
|
|
// The machine: what it is sent is applied, and its report says what runs is starting.
|
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
|
sequence := int64(0)
|
|
say := func(node, digest, state string) {
|
|
t.Helper()
|
|
sequence++
|
|
health := map[string]any{}
|
|
for k, v := range stated {
|
|
health[k] = v
|
|
}
|
|
health["at"] = time.Now().UTC().Format(time.RFC3339Nano)
|
|
if state != "" && stated != nil {
|
|
resources := []any{}
|
|
for _, r := range stated["resources"].([]any) {
|
|
kept := map[string]any{}
|
|
for k, v := range r.(map[string]any) {
|
|
kept[k] = v
|
|
}
|
|
kept["state"], kept["reason"] = state, ""
|
|
resources = append(resources, kept)
|
|
}
|
|
health["resources"] = resources
|
|
}
|
|
said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest,
|
|
"report_sequence": sequence}
|
|
if stated != nil {
|
|
said["health"] = health
|
|
}
|
|
body, _ := json.Marshal(said)
|
|
var report link.Report
|
|
if err := json.Unmarshal(body, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := listener.Heard(ctx, report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var sent [][]string
|
|
last := map[string]string{}
|
|
n := 0
|
|
wasSend := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
sent = append(sent, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
|
return nil, err
|
|
}
|
|
last[node] = digest
|
|
say(node, digest, "starting")
|
|
}
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = wasSend })
|
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
|
gateSettle, gateEvery = 0, 0
|
|
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
|
|
|
built := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
advancePlans(ctx, open) // the first machine is sent the new build, and starts it
|
|
if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) {
|
|
t.Fatalf("sent %v, not the first machine first", sent)
|
|
}
|
|
advancePlans(ctx, open) // judged while it starts
|
|
// Its container exits at start, and the runtime restarts it: the machine says so, again and again.
|
|
for i := 0; i < 6 && len(sent) == 1; i++ {
|
|
say("anchor", last["anchor"], "")
|
|
advancePlans(ctx, open)
|
|
}
|
|
gateBound = -time.Second // and the bound passes
|
|
advancePlans(ctx, open)
|
|
|
|
p, err := inv.PlanByID(ctx, "plan-crashloop")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gate := p.Modules["app"].Gate
|
|
for _, names := range sent {
|
|
if slices.Contains(names, "laptop") {
|
|
t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate)
|
|
}
|
|
}
|
|
if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed {
|
|
t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v",
|
|
p.State, p.Note, gate)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
|
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
|
}
|
|
}
|