Pass the controller's first-node gate on one reading after the witness window
mesh/merge-gate head 524295926f76 merged into trunk bff5b7f0e4f8 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 524295926f76 merged into trunk bff5b7f0e4f8 — fail: its merge-check.sh failed: --- FAIL: TestACoreBehindWhileTheNodeToolsSettleFailsNo…
mesh/delivery superseded: a newer head of the same pull request

The controller runs on one machine, so three readings over two minutes guarded no
other machine and held every walk behind it. Judge each module by its own rule:
one reading for the controller and the tool runner once the node-engine's witness
window has ended, one for mesh-delivery, three for the node-engine and the
resolver with the first two minutes after the send (novox/hq ADR 0351). The
gate's verdict names the rule it applied.
This commit is contained in:
2026-10-13 22:39:32 +02:00
parent bff5b7f0e4
commit 524295926f
11 changed files with 448 additions and 71 deletions
+236
View File
@@ -0,0 +1,236 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one
// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver
// keep three, the first two minutes after the send; the bus is kept as it was.
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
func withTheGatesTimes(t *testing.T) {
t.Helper()
wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
t.Cleanup(func() {
gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
})
}
func TestEachModuleHasItsGateRule(t *testing.T) {
withTheGatesTimes(t)
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
settle time.Duration
witnessed bool
said string
}{
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"app", nil, 1, 0, false, "a leaf module's rule"},
} {
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle,
c.witnessed, c.said)
}
}
}
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
func TestTheCoreStaysTheCore(t *testing.T) {
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
if !coreModule(m, catalogue.Manifest{Module: m}) {
t.Errorf("%s is no longer judged as the core for its class", m)
}
}
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
t.Error("the resolver's holder is no longer judged as the core for its class")
}
}
// gateOn judges one module on one machine, healthy at every reading.
type gateOn struct {
module, node string
g *inventory.PlanGate
holder *lease.Holder
}
func aGateOn(module, node string, since time.Time) *gateOn {
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
Applied: map[string]time.Time{}}}
}
func (o *gateOn) read(at time.Time, h health, why string) {
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
}
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(15 * time.Second)
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
// Ready, but inside the witness window, which opened before it took the lease: not counted.
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
o.read(sent.Add(after), healthGood, "")
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
}
}
o.read(sent.Add(80*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
}
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
!strings.Contains(o.g.Why, "ADR 0351") {
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
}
}
// A controller that starts again takes the lease again: its witness window starts again, and so does the
// gate's wait.
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(10 * time.Second)
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
o.read(sent.Add(90*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
}
o.read(sent.Add(130*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed {
t.Fatalf("not passed after the new window: %+v", o.g)
}
}
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
// and from the send at the least.
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
}
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
}
}
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
if o.g.Verdict != inventory.GateFailed {
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
}
o = aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != "" {
t.Fatalf("failed before the bound: %+v", o.g)
}
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
}
}
// The node-engine and the resolver keep three readings, the first at least two minutes after the send.
func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) {
withTheGatesTimes(t)
for _, module := range []string{"mesh-host", "dnsmasq"} {
sent := time.Now()
o := aGateOn(module, "ace", sent)
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
judge := func(after time.Duration) {
tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood},
worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}},
sent.Add(after))
}
for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} {
judge(after)
}
if o.g.Passes != 0 || o.g.Verdict != "" {
t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g)
}
judge(2 * time.Minute)
judge(160 * time.Second)
if o.g.Verdict != "" || o.g.Passes != 2 {
t.Fatalf("%s: after two readings %+v; want still judging", module, o.g)
}
judge(200 * time.Second)
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 ||
!strings.Contains(o.g.Why, "the core's rule: 3 readings") {
t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g)
}
}
}
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("node-tools", "ace", sent)
o.g.Applied["ace"] = sent.Add(10 * time.Second)
o.read(sent.Add(30*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
}
o.read(sent.Add(70*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
}
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
o.read(sent.Add(5*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
}
}
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
modules := []string{"mesh-controller", "mesh-host"}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
}
for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} {
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
}
if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 {
t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g)
}
}