Files
mesh-controller/cmd/mesh-controller/gate_followup_test.go
T
jochen a44dc01c65
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk
Excuse no wait for a move from a build not known, and count a module put back only once there is one (hq issue 318 review)
2026-10-08 15:49:07 +02:00

454 lines
20 KiB
Go

package main
import (
"context"
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
// wait, and a provider waiting for a login says who waits on it.
// withGateBounds sets the gate's bounds for one test.
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
t.Helper()
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery, gateBound = settle, every, bound
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
}
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
// of its modules' resources.
func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
return func() gateFacts {
now := time.Now()
at := now
return gateFacts{now: now,
reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{},
served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
}
}
}
func healthyContainer(module string) inventory.ResourceHealth {
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
State: link.StateHealthy}
}
// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
since := time.Now().Add(-time.Second)
base := factsOn(t, "laptop", since, healthyContainer("app"))
wasGather := gatherGateFacts
t.Cleanup(func() { gatherGateFacts = wasGather })
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
f := base()
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
return f, nil
}
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
judgings := 0
for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
t.Fatal(err)
}
judgings++
time.Sleep(30 * time.Millisecond)
}
if g.Verdict != inventory.GateFailed || judgings < gatePasses {
t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
judgings, g)
}
if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
}
if !strings.Contains(g.Why, "witness") {
t.Errorf("the verdict does not say what broke: %q", g.Why)
}
}
// broker_runtime is the node tools' module name, a core component a witness judges.
const broker_runtime = "node-tools"
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
// while a module beside it is not yet healthy counts the healthy one once.
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, 0, time.Hour, time.Hour)
since := time.Now().Add(-time.Minute)
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
wasGather := gatherGateFacts
t.Cleanup(func() { gatherGateFacts = wasGather })
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
now := time.Now()
for i := 0; i < 3; i++ {
if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
t.Fatal(err)
}
}
if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
}
}
// **A carried build's pass carries its own wait, never another's** (review (e)).
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
since := time.Now().Add(-time.Minute)
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
app, _, _ := inv.GateOf(ctx, "build-app-c2")
late, _, _ := inv.GateOf(ctx, "build-late-c2")
if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
t.Errorf("app's pass: %+v; want it without late's wait", app)
}
if !strings.Contains(late.Why, "relogin needed on laptop") {
t.Errorf("late's pass: %+v; want its own wait", late)
}
}
// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
for _, b := range []inventory.Build{
{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
b.Manifest = manifest
b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
Asked: b.Asked}); err != nil {
t.Fatal(err)
}
if b.Commit == "l1" {
if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
"late": "l1"}); err != nil {
t.Fatal(err)
}
}
}
wasMoves := machineMoves
t.Cleanup(func() { machineMoves = wasMoves })
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
gather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f, err := gather(ctx, open, component)
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
State: link.StateUnhealthy, Reason: "down"}}}}
return f, err
}
gateEvery, gateBound = 0, 400*time.Millisecond
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
advancePlans(ctx, g.open)
if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
break
}
time.Sleep(30 * time.Millisecond)
}
p := g.plan(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
}
if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
!strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
}
if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
t.Fatal("late's build is not marked failed at its gate")
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
current["late"].Commit)
}
}
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
k := conditionsFrom
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
t.Fatal(err)
}
at := h0
say := func(machine string, rs ...link.ResourceHealth) {
t.Helper()
at = at.Add(time.Second)
for i := range rs {
rs[i].Since = at
}
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
t.Fatal(err)
}
}
account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
for look := 0; look < 2; look++ {
say("anchor", account, unit)
say("laptop", shop)
}
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
var keys []string
var c conditions.Condition
for _, x := range list {
keys = append(keys, x.Key)
if x.Key == "module.db.anchor.relogin-needed" {
c = x
}
}
if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
}
if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
}
}
// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
// tools are marked and put back in the very judging that found them broken — their registered build is the
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
build := func(module, commit string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"app", broker_runtime} {
build(m, "c1", old)
if _, err := inv.Assign(ctx, "laptop", m); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
t.Fatal(err)
}
build("app", "c2", recent)
build(broker_runtime, "c2", recent)
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
})
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, _ := open.inventory.Assigned(ctx, node)
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
var sends []string
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
n++
carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
sends = append(sends, node+":"+carried[broker_runtime])
digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
var seen []string // the node tools' registered build at each judging
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
now := time.Now()
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, _ := open.inventory.LastReports(ctx)
for _, r := range reports {
f.reports[r.Node] = r
}
current, _ := open.inventory.CurrentBuilds(ctx)
seen = append(seen, current[broker_runtime].Commit)
app := healthyContainer("app")
if current[broker_runtime].Commit == "c2" {
// The witness reverted the node tools; app cannot reach them yet.
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
app.State, app.Reason = link.StateUnhealthy, "down"
}
f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
return f, nil
}
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
release := func() inventory.Plan {
t.Helper()
plans, _ := inv.RecentPlans(ctx, 10)
for _, p := range plans {
if p.Release != nil {
return p
}
}
t.Fatal("no walk")
return inventory.Plan{}
}
// Judged until the first judging has found the node tools broken, and one more.
for i := 0; i < 20 && len(seen) < 2; i++ {
advancePlans(ctx, open)
}
if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
}
if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
t.Fatal("the node tools' build is not marked failed at once")
}
if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
}
if !slices.Contains(sends, "laptop:c1") {
t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
}
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
advancePlans(ctx, open)
if release().State != inventory.PlanRolling {
break
}
time.Sleep(20 * time.Millisecond)
}
p := release()
if p.State != inventory.PlanFailed {
t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
}
if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v; want its own pass", v)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
}
}
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
record := func(commit string, m catalogue.Manifest) {
raw, _ := json.Marshal(m)
at := time.Now().Add(-time.Hour)
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil {
t.Fatal(err)
}
}
record("c1", catalogue.Manifest{Module: "lights"})
record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
"name": "operator", "groups": []any{"lights"}}}})
pairs := []judged{{module: "lights", node: "laptop"}}
shelf := map[string]catalogue.Manifest{}
for _, c := range []struct {
from string
want bool
}{{"c1", true}, {"", false}, {"c0-never-built", false}} {
g := &inventory.PlanGate{From: c.from, To: "c2"}
if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want {
t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want)
}
}
}