Pass the node-engine and resolver on their third reading, at most two minutes

The operator wants these gates as fast as they can be: counting readings only after
two minutes added about 80 s to every node-engine rollout for no safety. Count
from the send, pass on the third healthy reading, and pass a module healthy at
every reading at two minutes on what it has (novox/hq ADR 0352).

cmd/mesh-controller/gate_rule_test.go

@@ -10,19 +10,19 @@ import (
 	"github.com/novox/mesh-controller/internal/lease"
 )
 
-// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one
-// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver
-// keep three, the first two minutes after the send; the bus is kept as it was.
+// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
+// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
+// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
 
 // withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
 func withTheGatesTimes(t *testing.T) {
 	t.Helper()
-	wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow
+	wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
 	wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
-	gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
+	gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
 	gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
 	t.Cleanup(func() {
-		gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow
+		gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
 		gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
 	})
 }
@@ -33,7 +33,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
 		module    string
 		claims    []catalogue.Claim
 		passes    int
-		settle    time.Duration
+		within    time.Duration
 		witnessed bool
 		said      string
 	}{
@@ -46,8 +46,8 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
 		{"app", nil, 1, 0, false, "a leaf module's rule"},
 	} {
 		r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
-		if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
-			t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle,
+		if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
+			t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
 				c.witnessed, c.said)
 		}
 	}
@@ -161,38 +161,107 @@ func TestAControllerThatFailsIsStillFailed(t *testing.T) {
 	}
 }
 
-// The node-engine and the resolver keep three readings, the first at least two minutes after the send.
-func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) {
+// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
+func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
+	t.Helper()
+	g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
+	rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
+	floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
+	return g, func(after time.Duration, h health) {
+		why := ""
+		if h != healthGood {
+			why = "the node-engine reports another build"
+		}
+		tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
+			rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
+	}
+}
+
+// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
+// minutes (novox/hq ADR 0352).
+func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
 	withTheGatesTimes(t)
 	for _, module := range []string{"mesh-host", "dnsmasq"} {
-		sent := time.Now()
-		o := aGateOn(module, "ace", sent)
-		rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
-		floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
-		judge := func(after time.Duration) {
-			tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood},
-				worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}},
-				sent.Add(after))
+		g, judge := threeReadings(t, module, time.Now())
+		judge(15*time.Second, healthGood)
+		judge(55*time.Second, healthGood)
+		if g.Verdict != "" || g.Passes != 2 {
+			t.Fatalf("%s: after two readings %+v; want still judging", module, g)
 		}
-		for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} {
-			judge(after)
-		}
-		if o.g.Passes != 0 || o.g.Verdict != "" {
-			t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g)
-		}
-		judge(2 * time.Minute)
-		judge(160 * time.Second)
-		if o.g.Verdict != "" || o.g.Passes != 2 {
-			t.Fatalf("%s: after two readings %+v; want still judging", module, o.g)
-		}
-		judge(200 * time.Second)
-		if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 ||
-			!strings.Contains(o.g.Why, "the core's rule: 3 readings") {
-			t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g)
+		judge(95*time.Second, healthGood)
+		if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
+			!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
+			t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
 		}
 	}
 }
 
+// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
+// two minutes comes sooner than the spacing after the last pass.
+func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
+	withTheGatesTimes(t)
+	sent := time.Now()
+	last := sent.Add(100 * time.Second)
+	held := &inventory.PlanGate{Since: &sent, LastPass: &last}
+	if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
+		t.Fatal("the judging at two minutes is not due, or is due before")
+	}
+	held.Faulted = true
+	if dueAtTheMost(held, sent.Add(2*time.Minute)) {
+		t.Fatal("a send that was not healthy at a reading is judged at two minutes")
+	}
+
+	g, judge := threeReadings(t, "mesh-host", time.Now())
+	judge(100*time.Second, healthGood)
+	judge(2*time.Minute, healthGood)
+	if g.Verdict != inventory.GatePassed || g.Passes != 2 {
+		t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
+	}
+}
+
+// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
+func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
+	withTheGatesTimes(t)
+	g, judge := threeReadings(t, "mesh-host", time.Now())
+	judge(10*time.Second, healthNotYet)
+	judge(100*time.Second, healthGood)
+	judge(2*time.Minute, healthGood)
+	if g.Verdict != inventory.GatePassed || g.Faulted {
+		t.Fatalf("%+v; want passed at two minutes", g)
+	}
+}
+
+// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
+// never healthy fails at the bound and is put back; one found broken fails.
+func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
+	withTheGatesTimes(t)
+	g, judge := threeReadings(t, "mesh-host", time.Now())
+	judge(20*time.Second, healthNotYet)
+	judge(60*time.Second, healthGood)
+	judge(100*time.Second, healthGood)
+	judge(2*time.Minute+10*time.Second, healthGood)
+	if g.Verdict != inventory.GatePassed || g.Passes != 3 {
+		t.Fatalf("%+v; want passed only on its third healthy reading", g)
+	}
+	g, judge = threeReadings(t, "mesh-host", time.Now())
+	judge(60*time.Second, healthGood)
+	judge(100*time.Second, healthNotYet)
+	judge(2*time.Minute, healthGood)
+	if g.Verdict != "" || !g.Faulted {
+		t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
+	}
+	judge(10*time.Minute+time.Second, healthNotYet)
+	if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
+		t.Fatalf("never healthy again: %+v; want failed at the bound", g)
+	}
+	g, judge = threeReadings(t, "mesh-host", time.Now())
+	judge(60*time.Second, healthGood)
+	judge(100*time.Second, healthBroken)
+	if g.Verdict != inventory.GateFailed {
+		t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
+	}
+}
+
 // The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
 func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
 	withTheGatesTimes(t)
@@ -226,11 +295,11 @@ func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
 		rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
 		floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
 	}
-	for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} {
+	for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
 		tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
 			"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
 	}
-	if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 {
-		t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g)
+	if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
+		t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
 	}
 }