package main import ( "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) // The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass // on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's // resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so. // withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. func withTheGatesTimes(t *testing.T) { t.Helper() wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false t.Cleanup(func() { gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles }) } func TestEachModuleHasItsGateRule(t *testing.T) { withTheGatesTimes(t) for _, c := range []struct { module string claims []catalogue.Claim passes int within time.Duration witnessed bool said string }{ {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, {catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"}, {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, {"app", nil, 1, 0, false, "a leaf module's rule"}, } { r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within, c.witnessed, c.said) } } } // Every module that was core stays core for its delivery budget and its walk: only its readings change. func TestTheCoreStaysTheCore(t *testing.T) { for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { // The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 ยง3, issue 1210). claims := []catalogue.Claim(nil) if m == catalogue.DeliverySeat { claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} } if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) { t.Errorf("%s is no longer judged as the core for its class", m) } } if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { t.Error("the resolver's holder is no longer judged as the core for its class") } } // gateOn judges one module on one machine, healthy at every reading. type gateOn struct { module, node string g *inventory.PlanGate holder *lease.Holder } func aGateOn(module, node string, since time.Time) *gateOn { return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, Applied: map[string]time.Time{}}} } func (o *gateOn) read(at time.Time, h health, why string) { m := catalogue.Manifest{Module: o.module} if o.module == catalogue.DeliverySeat { // today's module, which claims the seat m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} } r := gateRuleOf(o.module, m) floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) } func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { withTheGatesTimes(t) sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) o := aGateOn("mesh-controller", "novox", sent) o.g.Applied["novox"] = sent.Add(15 * time.Second) o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} // Ready, but inside the witness window, which opened before it took the lease: not counted. for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { o.read(sent.Add(after), healthGood, "") if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) } } o.read(sent.Add(80*time.Second), healthGood, "") if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) } if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || !strings.Contains(o.g.Why, "ADR 0351") { t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) } } // A controller that starts again takes the lease again: its witness window starts again, and so does the // gate's wait. func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { withTheGatesTimes(t) sent := time.Now() o := aGateOn("mesh-controller", "novox", sent) o.g.Applied["novox"] = sent.Add(10 * time.Second) o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} o.read(sent.Add(90*time.Second), healthGood, "") if o.g.Verdict != "" { t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) } o.read(sent.Add(130*time.Second), healthGood, "") if o.g.Verdict != inventory.GatePassed { t.Fatalf("not passed after the new window: %+v", o.g) } } // Without the lease's word (a controller carried by another gate), the window counts from the applied report, // and from the send at the least. func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { withTheGatesTimes(t) sent := time.Now() r := gateRuleOf("mesh-controller", catalogue.Manifest{}) if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) } if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) } } // A controller that a witness put back, or that never becomes ready, fails its gate as before. func TestAControllerThatFailsIsStillFailed(t *testing.T) { withTheGatesTimes(t) sent := time.Now() o := aGateOn("mesh-controller", "novox", sent) o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") if o.g.Verdict != inventory.GateFailed { t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) } o = aGateOn("mesh-controller", "novox", sent) o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") if o.g.Verdict != "" { t.Fatalf("failed before the bound: %+v", o.g) } o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { t.Fatalf("never ready: %+v; want failed at the bound", o.g) } } // threeReadings judges a module under the three-reading rule on one machine, the send at sent. func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) { t.Helper() g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent} rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) return g, func(after time.Duration, h health) { why := "" if h != healthGood { why = "the node-engine reports another build" } tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after)) } } // A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two // minutes (novox/hq ADR 0352). func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) { withTheGatesTimes(t) for _, module := range []string{"mesh-host", "dnsmasq"} { g, judge := threeReadings(t, module, time.Now()) judge(15*time.Second, healthGood) judge(55*time.Second, healthGood) if g.Verdict != "" || g.Passes != 2 { t.Fatalf("%s: after two readings %+v; want still judging", module, g) } judge(95*time.Second, healthGood) if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 || !strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") { t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g) } } } // One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at // two minutes comes sooner than the spacing after the last pass. func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { withTheGatesTimes(t) sent := time.Now() last := sent.Add(100 * time.Second) held := &inventory.PlanGate{Since: &sent, LastPass: &last} if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) { t.Fatal("the judging at two minutes is not due, or is due before") } held.Faulted = true if dueAtTheMost(held, sent.Add(2*time.Minute)) { t.Fatal("a send that was not healthy at a reading is judged at two minutes") } g, judge := threeReadings(t, "mesh-host", time.Now()) judge(100*time.Second, healthGood) judge(2*time.Minute, healthGood) if g.Verdict != inventory.GatePassed || g.Passes != 2 { t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) } g, judge = threeReadings(t, "mesh-host", time.Now()) judge(2*time.Minute, healthGood) if g.Verdict != inventory.GatePassed || g.Passes != 1 { t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g) } } // A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) { withTheGatesTimes(t) g, judge := threeReadings(t, "mesh-host", time.Now()) judge(10*time.Second, healthNotYet) judge(100*time.Second, healthGood) judge(2*time.Minute, healthGood) if g.Verdict != inventory.GatePassed || g.Faulted { t.Fatalf("%+v; want passed at two minutes", g) } } // A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one // never healthy fails at the bound and is put back; one found broken fails. func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) { withTheGatesTimes(t) g, judge := threeReadings(t, "mesh-host", time.Now()) judge(20*time.Second, healthNotYet) judge(60*time.Second, healthGood) judge(100*time.Second, healthGood) judge(2*time.Minute+10*time.Second, healthGood) if g.Verdict != inventory.GatePassed || g.Passes != 3 { t.Fatalf("%+v; want passed only on its third healthy reading", g) } g, judge = threeReadings(t, "mesh-host", time.Now()) judge(60*time.Second, healthGood) judge(100*time.Second, healthNotYet) judge(2*time.Minute, healthGood) if g.Verdict != "" || !g.Faulted { t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g) } judge(10*time.Minute+time.Second, healthNotYet) if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") { t.Fatalf("never healthy again: %+v; want failed at the bound", g) } g, judge = threeReadings(t, "mesh-host", time.Now()) judge(60*time.Second, healthGood) judge(100*time.Second, healthBroken) if g.Verdict != inventory.GateFailed { t.Fatalf("a broken reading: %+v; want failed, to be put back", g) } } // The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { withTheGatesTimes(t) sent := time.Now() o := aGateOn("node-tools", "ace", sent) o.g.Applied["ace"] = sent.Add(10 * time.Second) o.read(sent.Add(30*time.Second), healthGood, "") if o.g.Verdict != "" { t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) } o.read(sent.Add(70*time.Second), healthGood, "") if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) } o = aGateOn(catalogue.DeliverySeat, "novox", sent) o.read(sent.Add(5*time.Second), healthGood, "") if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) } } // A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at // two minutes (novox/hq ADR 0352 decision 3). func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) { withTheGatesTimes(t) sent := time.Now() g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} modules := []string{"mesh-controller", "mesh-host"} rules, floors := map[string]gateRule{}, map[string]time.Time{} for _, m := range modules { rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) } floors["mesh-host"] = sent floors["mesh-controller"] = sent.Add(90 * time.Second) judge := func(after time.Duration, h health) { tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h}, worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after)) } judge(30*time.Second, healthGood) if g.Passes != 0 { t.Fatalf("a reading before the controller's floor was counted: %+v", g) } judge(70*time.Second, healthNotYet) judge(2*time.Minute, healthGood) if !g.Faulted || g.Verdict != "" { t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g) } } // A send carrying the controller and the node-engine is judged by the node-engine's three readings. func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { withTheGatesTimes(t) sent := time.Now() g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} modules := []string{"mesh-controller", "mesh-host"} rules, floors := map[string]gateRule{}, map[string]time.Time{} for _, m := range modules { rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) } for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} { tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) } if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 { t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g) } }