jschoubben
675466a8c3
mesh/merge-gate head 675466a8c35f merged into trunk bff5b7f0e4f8 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 675466a8c35f merged into trunk bff5b7f0e4f8 — pass: its merge-check.sh passed
mesh/delivery delivered
Two tier tests relied on the two-minute settle to keep a module unjudged; the witness window and the gate's spacing now hold it. A healthy reading taken before the controller's floor counted no pass, so a dip after it did not take the two-minute pass away (novox/hq ADR 0352 decision 3).
339 lines
15 KiB
Go
339 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
)
|
|
|
|
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
|
|
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
|
|
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
|
|
|
|
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
|
|
func withTheGatesTimes(t *testing.T) {
|
|
t.Helper()
|
|
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
|
|
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
|
|
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
|
|
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
|
|
t.Cleanup(func() {
|
|
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
|
|
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
|
|
})
|
|
}
|
|
|
|
func TestEachModuleHasItsGateRule(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
for _, c := range []struct {
|
|
module string
|
|
claims []catalogue.Claim
|
|
passes int
|
|
within time.Duration
|
|
witnessed bool
|
|
said string
|
|
}{
|
|
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
|
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
|
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
|
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
|
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
|
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
|
{"app", nil, 1, 0, false, "a leaf module's rule"},
|
|
} {
|
|
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
|
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
|
|
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
|
|
c.witnessed, c.said)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
|
func TestTheCoreStaysTheCore(t *testing.T) {
|
|
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
|
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
|
t.Errorf("%s is no longer judged as the core for its class", m)
|
|
}
|
|
}
|
|
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
|
|
t.Error("the resolver's holder is no longer judged as the core for its class")
|
|
}
|
|
}
|
|
|
|
// gateOn judges one module on one machine, healthy at every reading.
|
|
type gateOn struct {
|
|
module, node string
|
|
g *inventory.PlanGate
|
|
holder *lease.Holder
|
|
}
|
|
|
|
func aGateOn(module, node string, since time.Time) *gateOn {
|
|
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
|
|
Applied: map[string]time.Time{}}}
|
|
}
|
|
|
|
func (o *gateOn) read(at time.Time, h health, why string) {
|
|
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
|
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
|
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
|
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
|
}
|
|
|
|
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
|
o := aGateOn("mesh-controller", "novox", sent)
|
|
o.g.Applied["novox"] = sent.Add(15 * time.Second)
|
|
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
|
|
|
|
// Ready, but inside the witness window, which opened before it took the lease: not counted.
|
|
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
|
|
o.read(sent.Add(after), healthGood, "")
|
|
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
|
|
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
|
|
}
|
|
}
|
|
o.read(sent.Add(80*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
|
|
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
|
|
}
|
|
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
|
|
!strings.Contains(o.g.Why, "ADR 0351") {
|
|
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
|
|
}
|
|
}
|
|
|
|
// A controller that starts again takes the lease again: its witness window starts again, and so does the
|
|
// gate's wait.
|
|
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
o := aGateOn("mesh-controller", "novox", sent)
|
|
o.g.Applied["novox"] = sent.Add(10 * time.Second)
|
|
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
|
|
o.read(sent.Add(90*time.Second), healthGood, "")
|
|
if o.g.Verdict != "" {
|
|
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
|
|
}
|
|
o.read(sent.Add(130*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed {
|
|
t.Fatalf("not passed after the new window: %+v", o.g)
|
|
}
|
|
}
|
|
|
|
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
|
|
// and from the send at the least.
|
|
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
|
|
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
|
|
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
|
|
}
|
|
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
|
|
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
|
|
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
|
|
}
|
|
}
|
|
|
|
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
|
|
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
o := aGateOn("mesh-controller", "novox", sent)
|
|
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
|
|
if o.g.Verdict != inventory.GateFailed {
|
|
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
|
|
}
|
|
o = aGateOn("mesh-controller", "novox", sent)
|
|
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
|
|
if o.g.Verdict != "" {
|
|
t.Fatalf("failed before the bound: %+v", o.g)
|
|
}
|
|
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
|
|
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
|
|
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
|
|
}
|
|
}
|
|
|
|
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
|
|
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
|
|
t.Helper()
|
|
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
|
|
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
|
|
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
|
|
return g, func(after time.Duration, h health) {
|
|
why := ""
|
|
if h != healthGood {
|
|
why = "the node-engine reports another build"
|
|
}
|
|
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
|
|
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
|
|
}
|
|
}
|
|
|
|
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
|
|
// minutes (novox/hq ADR 0352).
|
|
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
for _, module := range []string{"mesh-host", "dnsmasq"} {
|
|
g, judge := threeReadings(t, module, time.Now())
|
|
judge(15*time.Second, healthGood)
|
|
judge(55*time.Second, healthGood)
|
|
if g.Verdict != "" || g.Passes != 2 {
|
|
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
|
|
}
|
|
judge(95*time.Second, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
|
|
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
|
|
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
|
|
// two minutes comes sooner than the spacing after the last pass.
|
|
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
last := sent.Add(100 * time.Second)
|
|
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
|
|
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
|
t.Fatal("the judging at two minutes is not due, or is due before")
|
|
}
|
|
held.Faulted = true
|
|
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
|
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
|
|
}
|
|
|
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
|
judge(100*time.Second, healthGood)
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
|
|
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
|
|
}
|
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
|
|
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
|
|
}
|
|
}
|
|
|
|
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
|
|
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
|
judge(10*time.Second, healthNotYet)
|
|
judge(100*time.Second, healthGood)
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Faulted {
|
|
t.Fatalf("%+v; want passed at two minutes", g)
|
|
}
|
|
}
|
|
|
|
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
|
|
// never healthy fails at the bound and is put back; one found broken fails.
|
|
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
|
judge(20*time.Second, healthNotYet)
|
|
judge(60*time.Second, healthGood)
|
|
judge(100*time.Second, healthGood)
|
|
judge(2*time.Minute+10*time.Second, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
|
|
t.Fatalf("%+v; want passed only on its third healthy reading", g)
|
|
}
|
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
|
judge(60*time.Second, healthGood)
|
|
judge(100*time.Second, healthNotYet)
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != "" || !g.Faulted {
|
|
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
|
|
}
|
|
judge(10*time.Minute+time.Second, healthNotYet)
|
|
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
|
|
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
|
|
}
|
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
|
judge(60*time.Second, healthGood)
|
|
judge(100*time.Second, healthBroken)
|
|
if g.Verdict != inventory.GateFailed {
|
|
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
|
|
}
|
|
}
|
|
|
|
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
|
|
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
o := aGateOn("node-tools", "ace", sent)
|
|
o.g.Applied["ace"] = sent.Add(10 * time.Second)
|
|
o.read(sent.Add(30*time.Second), healthGood, "")
|
|
if o.g.Verdict != "" {
|
|
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
|
|
}
|
|
o.read(sent.Add(70*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
|
|
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
|
|
}
|
|
|
|
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
|
|
o.read(sent.Add(5*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
|
|
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
|
|
}
|
|
}
|
|
|
|
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
|
|
// two minutes (novox/hq ADR 0352 decision 3).
|
|
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
|
modules := []string{"mesh-controller", "mesh-host"}
|
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
|
for _, m := range modules {
|
|
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
|
}
|
|
floors["mesh-host"] = sent
|
|
floors["mesh-controller"] = sent.Add(90 * time.Second)
|
|
judge := func(after time.Duration, h health) {
|
|
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
|
|
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
|
|
}
|
|
judge(30*time.Second, healthGood)
|
|
if g.Passes != 0 {
|
|
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
|
|
}
|
|
judge(70*time.Second, healthNotYet)
|
|
judge(2*time.Minute, healthGood)
|
|
if !g.Faulted || g.Verdict != "" {
|
|
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
|
|
}
|
|
}
|
|
|
|
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
|
|
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
|
modules := []string{"mesh-controller", "mesh-host"}
|
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
|
for _, m := range modules {
|
|
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
|
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
|
|
}
|
|
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
|
|
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
|
|
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
|
|
}
|
|
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
|
|
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
|
|
}
|
|
}
|