mesh-controller/cmd/mesh-controller/gate_rule_test.go

package main

import (
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/lease"
)

// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.

// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
func withTheGatesTimes(t *testing.T) {
	t.Helper()
	wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
	wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
	gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
	gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
	t.Cleanup(func() {
		gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
		gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
	})
}

func TestEachModuleHasItsGateRule(t *testing.T) {
	withTheGatesTimes(t)
	for _, c := range []struct {
		module    string
		claims    []catalogue.Claim
		passes    int
		within    time.Duration
		witnessed bool
		said      string
	}{
		{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
		{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
		{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
		{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
		{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
		{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
		{"app", nil, 1, 0, false, "a leaf module's rule"},
	} {
		r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
		if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
			t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
				c.witnessed, c.said)
		}
	}
}

// Every module that was core stays core for its delivery budget and its walk: only its readings change.
func TestTheCoreStaysTheCore(t *testing.T) {
	for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
		if !coreModule(m, catalogue.Manifest{Module: m}) {
			t.Errorf("%s is no longer judged as the core for its class", m)
		}
	}
	if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
		t.Error("the resolver's holder is no longer judged as the core for its class")
	}
}

// gateOn judges one module on one machine, healthy at every reading.
type gateOn struct {
	module, node string
	g            *inventory.PlanGate
	holder       *lease.Holder
}

func aGateOn(module, node string, since time.Time) *gateOn {
	return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
		Applied: map[string]time.Time{}}}
}

func (o *gateOn) read(at time.Time, h health, why string) {
	r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
	floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
	tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
		worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
}

func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
	o := aGateOn("mesh-controller", "novox", sent)
	o.g.Applied["novox"] = sent.Add(15 * time.Second)
	o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}

	// Ready, but inside the witness window, which opened before it took the lease: not counted.
	for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
		o.read(sent.Add(after), healthGood, "")
		if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
			t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
		}
	}
	o.read(sent.Add(80*time.Second), healthGood, "")
	if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
		t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
	}
	if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
		!strings.Contains(o.g.Why, "ADR 0351") {
		t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
	}
}

// A controller that starts again takes the lease again: its witness window starts again, and so does the
// gate's wait.
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	o := aGateOn("mesh-controller", "novox", sent)
	o.g.Applied["novox"] = sent.Add(10 * time.Second)
	o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
	o.read(sent.Add(90*time.Second), healthGood, "")
	if o.g.Verdict != "" {
		t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
	}
	o.read(sent.Add(130*time.Second), healthGood, "")
	if o.g.Verdict != inventory.GatePassed {
		t.Fatalf("not passed after the new window: %+v", o.g)
	}
}

// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
// and from the send at the least.
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	r := gateRuleOf("mesh-controller", catalogue.Manifest{})
	if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
		t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
	}
	if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
		map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
		t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
	}
}

// A controller that a witness put back, or that never becomes ready, fails its gate as before.
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	o := aGateOn("mesh-controller", "novox", sent)
	o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
	if o.g.Verdict != inventory.GateFailed {
		t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
	}
	o = aGateOn("mesh-controller", "novox", sent)
	o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
	if o.g.Verdict != "" {
		t.Fatalf("failed before the bound: %+v", o.g)
	}
	o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
	if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
		t.Fatalf("never ready: %+v; want failed at the bound", o.g)
	}
}

// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
	t.Helper()
	g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
	rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
	floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
	return g, func(after time.Duration, h health) {
		why := ""
		if h != healthGood {
			why = "the node-engine reports another build"
		}
		tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
			rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
	}
}

// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
// minutes (novox/hq ADR 0352).
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
	withTheGatesTimes(t)
	for _, module := range []string{"mesh-host", "dnsmasq"} {
		g, judge := threeReadings(t, module, time.Now())
		judge(15*time.Second, healthGood)
		judge(55*time.Second, healthGood)
		if g.Verdict != "" || g.Passes != 2 {
			t.Fatalf("%s: after two readings %+v; want still judging", module, g)
		}
		judge(95*time.Second, healthGood)
		if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
			!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
			t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
		}
	}
}

// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
// two minutes comes sooner than the spacing after the last pass.
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	last := sent.Add(100 * time.Second)
	held := &inventory.PlanGate{Since: &sent, LastPass: &last}
	if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
		t.Fatal("the judging at two minutes is not due, or is due before")
	}
	held.Faulted = true
	if dueAtTheMost(held, sent.Add(2*time.Minute)) {
		t.Fatal("a send that was not healthy at a reading is judged at two minutes")
	}

	g, judge := threeReadings(t, "mesh-host", time.Now())
	judge(100*time.Second, healthGood)
	judge(2*time.Minute, healthGood)
	if g.Verdict != inventory.GatePassed || g.Passes != 2 {
		t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
	}
	g, judge = threeReadings(t, "mesh-host", time.Now())
	judge(2*time.Minute, healthGood)
	if g.Verdict != inventory.GatePassed || g.Passes != 1 {
		t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
	}
}

// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
	withTheGatesTimes(t)
	g, judge := threeReadings(t, "mesh-host", time.Now())
	judge(10*time.Second, healthNotYet)
	judge(100*time.Second, healthGood)
	judge(2*time.Minute, healthGood)
	if g.Verdict != inventory.GatePassed || g.Faulted {
		t.Fatalf("%+v; want passed at two minutes", g)
	}
}

// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
// never healthy fails at the bound and is put back; one found broken fails.
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
	withTheGatesTimes(t)
	g, judge := threeReadings(t, "mesh-host", time.Now())
	judge(20*time.Second, healthNotYet)
	judge(60*time.Second, healthGood)
	judge(100*time.Second, healthGood)
	judge(2*time.Minute+10*time.Second, healthGood)
	if g.Verdict != inventory.GatePassed || g.Passes != 3 {
		t.Fatalf("%+v; want passed only on its third healthy reading", g)
	}
	g, judge = threeReadings(t, "mesh-host", time.Now())
	judge(60*time.Second, healthGood)
	judge(100*time.Second, healthNotYet)
	judge(2*time.Minute, healthGood)
	if g.Verdict != "" || !g.Faulted {
		t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
	}
	judge(10*time.Minute+time.Second, healthNotYet)
	if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
		t.Fatalf("never healthy again: %+v; want failed at the bound", g)
	}
	g, judge = threeReadings(t, "mesh-host", time.Now())
	judge(60*time.Second, healthGood)
	judge(100*time.Second, healthBroken)
	if g.Verdict != inventory.GateFailed {
		t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
	}
}

// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	o := aGateOn("node-tools", "ace", sent)
	o.g.Applied["ace"] = sent.Add(10 * time.Second)
	o.read(sent.Add(30*time.Second), healthGood, "")
	if o.g.Verdict != "" {
		t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
	}
	o.read(sent.Add(70*time.Second), healthGood, "")
	if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
		t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
	}

	o = aGateOn(catalogue.DeliverySeat, "novox", sent)
	o.read(sent.Add(5*time.Second), healthGood, "")
	if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
		t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
	}
}

// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
// two minutes (novox/hq ADR 0352 decision 3).
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
	modules := []string{"mesh-controller", "mesh-host"}
	rules, floors := map[string]gateRule{}, map[string]time.Time{}
	for _, m := range modules {
		rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
	}
	floors["mesh-host"] = sent
	floors["mesh-controller"] = sent.Add(90 * time.Second)
	judge := func(after time.Duration, h health) {
		tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
			worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
	}
	judge(30*time.Second, healthGood)
	if g.Passes != 0 {
		t.Fatalf("a reading before the controller's floor was counted: %+v", g)
	}
	judge(70*time.Second, healthNotYet)
	judge(2*time.Minute, healthGood)
	if !g.Faulted || g.Verdict != "" {
		t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
	}
}

// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
	withTheGatesTimes(t)
	sent := time.Now()
	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
	modules := []string{"mesh-controller", "mesh-host"}
	rules, floors := map[string]gateRule{}, map[string]time.Time{}
	for _, m := range modules {
		rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
		floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
	}
	for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
		tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
			"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
	}
	if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
		t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
	}
}