Pass the node-engine and resolver on their third reading, at most two minutes

The operator wants these gates as fast as they can be: counting readings only after
two minutes added about 80 s to every node-engine rollout for no safety. Count
from the send, pass on the third healthy reading, and pass a module healthy at
every reading at two minutes on what it has (novox/hq ADR 0352).

cmd/mesh-controller/gate.go

@@ -28,10 +28,11 @@ import (
 // that machine reported it applied. A build that applies and then does nothing, crashes, serves no
 // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
 // the component's health — the core's health definitions, or a module's own — within ten minutes of the
-// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the
-// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the
+// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
+// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
 // node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
-// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3).
+// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
+// decision 3).
 // Only then are the rest sent.
 //
 // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
@@ -43,12 +44,14 @@ import (
 
 // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
 var (
-	// gateSettle is how long after the send a core module judged by three readings is first read, at the least.
-	gateSettle = 2 * time.Minute
+	// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
+	// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
+	// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
+	gateWithin = 2 * time.Minute
 	// gateBound is how long after the apply the build may take to become healthy.
 	gateBound = 10 * time.Minute
 	// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
-	// the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the
+	// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
 	// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
 	// decision 4 as ADR 0304 decision 3 changed it, issue 524).
 	gatePasses      = 3
@@ -78,11 +81,12 @@ func coreModule(module string, m catalogue.Manifest) bool {
 }
 
 // gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
-// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's
-// witness window for it. said names the rule in the gate's text.
+// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
+// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
+// (novox/hq ADR 0352). said names the rule in the gate's text.
 type gateRule struct {
 	passes    int
-	settle    time.Duration
+	within    time.Duration
 	witnessed bool
 	said      string
 }
@@ -105,27 +109,27 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule {
 	case module == catalogue.DeliverySeat:
 		return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
 	case coreModule(module, m):
-		return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+
-			"the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)}
+		return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
+			"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
 	}
 	r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
 	if gateLeafSettles {
-		r.settle = gateSettle
-		r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send",
-			gateLeafPasses, gateSettle)
+		r.within = gateWithin
+		r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
+			gateLeafPasses, gateWithin)
 	}
 	return r
 }
 
-// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed
-// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1
-// and 4). The window opens when the node-engine starts the build, which is before the machine reports it
+// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
+// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
+// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
 // applied (applied, as the gate first read it) and, for the controller, before the new controller took the
 // lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
 // A controller that starts again takes the lease again, and so moves its floor on. Pure.
 func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
 	holder *lease.Holder) time.Time {
-	floor := since.Add(rule.settle)
+	floor := since
 	if !rule.witnessed {
 		return floor
 	}
@@ -568,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
 	return judgeMoves(ctx, open, g, pairs, now)
 }
 
+// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
+// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
+func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
+	most := g.Since.Add(gateWithin)
+	return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
+}
+
 // judged is one module on one machine, as a gate judges it.
 type judged struct{ module, node string }
 
@@ -578,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	if g.Verdict != "" {
 		return g.Verdict, nil
 	}
-	if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
+	// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
+	// a module healthy throughout is never held past it (novox/hq ADR 0352).
+	if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
 		return "", nil
 	}
 	for _, c := range g.Carried {
@@ -741,8 +754,8 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 			}
 			g.Waits[m] = w
 		}
-		// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the
-		// three readings' first two minutes after the send, the controller's one after the witness window.
+		// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
+		// controller's and the tool runner's after the witness window, any other module's from the send.
 		if now.Before(j.floors[m]) {
 			continue
 		}
@@ -754,12 +767,13 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 		}
 	}
 	// What the send needs is what its most demanding module needs — the most readings, then the latest
-	// floor: a send carrying the node-engine is judged by its three readings.
-	wants, floor, said := 0, time.Time{}, ""
+	// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
+	// its two minutes (novox/hq ADR 0352).
+	wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
 	for _, m := range modules {
 		r, f := j.rules[m], j.floors[m]
 		if r.passes > wants || (r.passes == wants && f.After(floor)) {
-			wants, said = r.passes, r.said
+			wants, within, said = r.passes, r.within, r.said
 		}
 		if f.After(floor) {
 			floor = f
@@ -768,14 +782,19 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 	if len(modules) == 0 {
 		// A send judged on no machine of its modules (its first machine not running them) has no class to
 		// be judged by: it is judged as the core, never passed at once without a reading.
-		wants, floor = gatePasses, g.Since.Add(gateSettle)
-		said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle)
+		wants, floor, within = gatePasses, *g.Since, gateWithin
+		said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
+	}
+	// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
+	heldLongEnough := func(within time.Duration) bool {
+		return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
 	}
 	g.Wants = wants
 	// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
 	// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
 	passedAlone := func(m string) bool {
-		return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes
+		r := j.rules[m]
+		return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
 	}
 	// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
 	fail := func(why string) {
@@ -792,6 +811,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 	pastBound := now.Sub(*g.Since) > gateBound
 	switch {
 	case worst == healthBroken:
+		g.Faulted = true
 		g.Read(now, false, g.BrokenWhy)
 		var judging []string
 		for _, m := range modules {
@@ -810,9 +830,11 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 	case worst == healthWaiting:
 		// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
 		// the provider's own condition says what is wrong (ADR 0240 rule 5).
+		g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		g.Read(now, false, why)
 	case worst == healthNotYet:
+		g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		g.Read(now, false, why)
 		if pastBound {
@@ -831,7 +853,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
 		g.Read(now, true, "")
-		if g.Passes >= wants {
+		if g.Passes >= wants || heldLongEnough(within) {
 			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
 				now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
 		}