mesh-tools
013dad4688
Swaps three import paths for mesh-sdk's go/catalogue, go/overlay and go/validate (the same packages, copied from this repository at 6c2a8d62) and lets gofmt order the groups. The six swap commits compile together once the pin commit (go.mod, modules.txt) is on the branch; each alone does not, because a file on the SDK's types cannot call one still on the old package.
348 lines
16 KiB
Go
348 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
)
|
|
|
|
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
|
|
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
|
|
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
|
|
|
|
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
|
|
func withTheGatesTimes(t *testing.T) {
|
|
t.Helper()
|
|
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
|
|
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
|
|
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
|
|
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
|
|
t.Cleanup(func() {
|
|
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
|
|
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
|
|
})
|
|
}
|
|
|
|
func TestEachModuleHasItsGateRule(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
for _, c := range []struct {
|
|
module string
|
|
claims []catalogue.Claim
|
|
passes int
|
|
within time.Duration
|
|
witnessed bool
|
|
said string
|
|
}{
|
|
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
|
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
|
{catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"},
|
|
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
|
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
|
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
|
{"app", nil, 1, 0, false, "a leaf module's rule"},
|
|
} {
|
|
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
|
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
|
|
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
|
|
c.witnessed, c.said)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
|
func TestTheCoreStaysTheCore(t *testing.T) {
|
|
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
|
// The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210).
|
|
claims := []catalogue.Claim(nil)
|
|
if m == catalogue.DeliverySeat {
|
|
claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
|
}
|
|
if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) {
|
|
t.Errorf("%s is no longer judged as the core for its class", m)
|
|
}
|
|
}
|
|
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
|
|
t.Error("the resolver's holder is no longer judged as the core for its class")
|
|
}
|
|
}
|
|
|
|
// gateOn judges one module on one machine, healthy at every reading.
|
|
type gateOn struct {
|
|
module, node string
|
|
g *inventory.PlanGate
|
|
holder *lease.Holder
|
|
}
|
|
|
|
func aGateOn(module, node string, since time.Time) *gateOn {
|
|
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
|
|
Applied: map[string]time.Time{}}}
|
|
}
|
|
|
|
func (o *gateOn) read(at time.Time, h health, why string) {
|
|
m := catalogue.Manifest{Module: o.module}
|
|
if o.module == catalogue.DeliverySeat { // today's module, which claims the seat
|
|
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
|
}
|
|
r := gateRuleOf(o.module, m)
|
|
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
|
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
|
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
|
}
|
|
|
|
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
|
o := aGateOn("mesh-controller", "novox", sent)
|
|
o.g.Applied["novox"] = sent.Add(15 * time.Second)
|
|
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
|
|
|
|
// Ready, but inside the witness window, which opened before it took the lease: not counted.
|
|
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
|
|
o.read(sent.Add(after), healthGood, "")
|
|
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
|
|
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
|
|
}
|
|
}
|
|
o.read(sent.Add(80*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
|
|
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
|
|
}
|
|
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
|
|
!strings.Contains(o.g.Why, "ADR 0351") {
|
|
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
|
|
}
|
|
}
|
|
|
|
// A controller that starts again takes the lease again: its witness window starts again, and so does the
|
|
// gate's wait.
|
|
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
o := aGateOn("mesh-controller", "novox", sent)
|
|
o.g.Applied["novox"] = sent.Add(10 * time.Second)
|
|
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
|
|
o.read(sent.Add(90*time.Second), healthGood, "")
|
|
if o.g.Verdict != "" {
|
|
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
|
|
}
|
|
o.read(sent.Add(130*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed {
|
|
t.Fatalf("not passed after the new window: %+v", o.g)
|
|
}
|
|
}
|
|
|
|
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
|
|
// and from the send at the least.
|
|
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
|
|
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
|
|
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
|
|
}
|
|
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
|
|
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
|
|
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
|
|
}
|
|
}
|
|
|
|
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
|
|
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
o := aGateOn("mesh-controller", "novox", sent)
|
|
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
|
|
if o.g.Verdict != inventory.GateFailed {
|
|
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
|
|
}
|
|
o = aGateOn("mesh-controller", "novox", sent)
|
|
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
|
|
if o.g.Verdict != "" {
|
|
t.Fatalf("failed before the bound: %+v", o.g)
|
|
}
|
|
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
|
|
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
|
|
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
|
|
}
|
|
}
|
|
|
|
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
|
|
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
|
|
t.Helper()
|
|
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
|
|
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
|
|
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
|
|
return g, func(after time.Duration, h health) {
|
|
why := ""
|
|
if h != healthGood {
|
|
why = "the node-engine reports another build"
|
|
}
|
|
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
|
|
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
|
|
}
|
|
}
|
|
|
|
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
|
|
// minutes (novox/hq ADR 0352).
|
|
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
for _, module := range []string{"mesh-host", "dnsmasq"} {
|
|
g, judge := threeReadings(t, module, time.Now())
|
|
judge(15*time.Second, healthGood)
|
|
judge(55*time.Second, healthGood)
|
|
if g.Verdict != "" || g.Passes != 2 {
|
|
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
|
|
}
|
|
judge(95*time.Second, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
|
|
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
|
|
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
|
|
// two minutes comes sooner than the spacing after the last pass.
|
|
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
last := sent.Add(100 * time.Second)
|
|
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
|
|
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
|
t.Fatal("the judging at two minutes is not due, or is due before")
|
|
}
|
|
held.Faulted = true
|
|
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
|
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
|
|
}
|
|
|
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
|
judge(100*time.Second, healthGood)
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
|
|
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
|
|
}
|
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
|
|
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
|
|
}
|
|
}
|
|
|
|
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
|
|
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
|
judge(10*time.Second, healthNotYet)
|
|
judge(100*time.Second, healthGood)
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Faulted {
|
|
t.Fatalf("%+v; want passed at two minutes", g)
|
|
}
|
|
}
|
|
|
|
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
|
|
// never healthy fails at the bound and is put back; one found broken fails.
|
|
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
|
judge(20*time.Second, healthNotYet)
|
|
judge(60*time.Second, healthGood)
|
|
judge(100*time.Second, healthGood)
|
|
judge(2*time.Minute+10*time.Second, healthGood)
|
|
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
|
|
t.Fatalf("%+v; want passed only on its third healthy reading", g)
|
|
}
|
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
|
judge(60*time.Second, healthGood)
|
|
judge(100*time.Second, healthNotYet)
|
|
judge(2*time.Minute, healthGood)
|
|
if g.Verdict != "" || !g.Faulted {
|
|
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
|
|
}
|
|
judge(10*time.Minute+time.Second, healthNotYet)
|
|
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
|
|
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
|
|
}
|
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
|
judge(60*time.Second, healthGood)
|
|
judge(100*time.Second, healthBroken)
|
|
if g.Verdict != inventory.GateFailed {
|
|
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
|
|
}
|
|
}
|
|
|
|
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
|
|
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
o := aGateOn("node-tools", "ace", sent)
|
|
o.g.Applied["ace"] = sent.Add(10 * time.Second)
|
|
o.read(sent.Add(30*time.Second), healthGood, "")
|
|
if o.g.Verdict != "" {
|
|
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
|
|
}
|
|
o.read(sent.Add(70*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
|
|
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
|
|
}
|
|
|
|
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
|
|
o.read(sent.Add(5*time.Second), healthGood, "")
|
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
|
|
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
|
|
}
|
|
}
|
|
|
|
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
|
|
// two minutes (novox/hq ADR 0352 decision 3).
|
|
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
|
modules := []string{"mesh-controller", "mesh-host"}
|
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
|
for _, m := range modules {
|
|
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
|
}
|
|
floors["mesh-host"] = sent
|
|
floors["mesh-controller"] = sent.Add(90 * time.Second)
|
|
judge := func(after time.Duration, h health) {
|
|
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
|
|
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
|
|
}
|
|
judge(30*time.Second, healthGood)
|
|
if g.Passes != 0 {
|
|
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
|
|
}
|
|
judge(70*time.Second, healthNotYet)
|
|
judge(2*time.Minute, healthGood)
|
|
if !g.Faulted || g.Verdict != "" {
|
|
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
|
|
}
|
|
}
|
|
|
|
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
|
|
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
|
|
withTheGatesTimes(t)
|
|
sent := time.Now()
|
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
|
modules := []string{"mesh-controller", "mesh-host"}
|
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
|
for _, m := range modules {
|
|
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
|
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
|
|
}
|
|
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
|
|
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
|
|
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
|
|
}
|
|
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
|
|
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
|
|
}
|
|
}
|