Pass the node-engine and resolver on their third reading, at most two minutes

The operator wants these gates as fast as they can be: counting readings only after
two minutes added about 80 s to every node-engine rollout for no safety. Count
from the send, pass on the third healthy reading, and pass a module healthy at
every reading at two minutes on what it has (novox/hq ADR 0352).

cmd/mesh-controller/found_wait_test.go

@@ -81,13 +81,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
 				}
 				return f, err
 			}
-			wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+			wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 			// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 			wasWindow := gateWitnessWindow
 			gateWitnessWindow = 0
 			t.Cleanup(func() { gateWitnessWindow = wasWindow })
-			t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+			t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
-			gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
+			gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
 			deadline := time.Now().Add(5 * time.Second)
 			for time.Now().Before(deadline) {
 				advancePlans(ctx, b.open)

cmd/mesh-controller/gate.go

@@ -28,10 +28,11 @@ import (
 // that machine reported it applied. A build that applies and then does nothing, crashes, serves no
 // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
 // the component's health — the core's health definitions, or a module's own — within ten minutes of the
-// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the
+// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
-// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the
+// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
 // node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
-// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3).
+// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
+// decision 3).
 // Only then are the rest sent.
 //
 // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
@@ -43,12 +44,14 @@ import (
 
 // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
 var (
-	// gateSettle is how long after the send a core module judged by three readings is first read, at the least.
+	// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
-	gateSettle = 2 * time.Minute
+	// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
+	// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
+	gateWithin = 2 * time.Minute
 	// gateBound is how long after the apply the build may take to become healthy.
 	gateBound = 10 * time.Minute
 	// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
-	// the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the
+	// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
 	// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
 	// decision 4 as ADR 0304 decision 3 changed it, issue 524).
 	gatePasses      = 3
@@ -78,11 +81,12 @@ func coreModule(module string, m catalogue.Manifest) bool {
 }
 
 // gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
-// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's
+// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
-// witness window for it. said names the rule in the gate's text.
+// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
+// (novox/hq ADR 0352). said names the rule in the gate's text.
 type gateRule struct {
 	passes    int
-	settle    time.Duration
+	within    time.Duration
 	witnessed bool
 	said      string
 }
@@ -105,27 +109,27 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule {
 	case module == catalogue.DeliverySeat:
 		return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
 	case coreModule(module, m):
-		return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+
+		return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
-			"the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)}
+			"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
 	}
 	r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
 	if gateLeafSettles {
-		r.settle = gateSettle
+		r.within = gateWithin
-		r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send",
+		r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
-			gateLeafPasses, gateSettle)
+			gateLeafPasses, gateWithin)
 	}
 	return r
 }
 
-// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed
+// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
-// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1
+// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
-// and 4). The window opens when the node-engine starts the build, which is before the machine reports it
+// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
 // applied (applied, as the gate first read it) and, for the controller, before the new controller took the
 // lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
 // A controller that starts again takes the lease again, and so moves its floor on. Pure.
 func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
 	holder *lease.Holder) time.Time {
-	floor := since.Add(rule.settle)
+	floor := since
 	if !rule.witnessed {
 		return floor
 	}
@@ -568,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
 	return judgeMoves(ctx, open, g, pairs, now)
 }
 
+// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
+// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
+func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
+	most := g.Since.Add(gateWithin)
+	return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
+}
+
 // judged is one module on one machine, as a gate judges it.
 type judged struct{ module, node string }
 
@@ -578,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	if g.Verdict != "" {
 		return g.Verdict, nil
 	}
-	if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
+	// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
+	// a module healthy throughout is never held past it (novox/hq ADR 0352).
+	if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
 		return "", nil
 	}
 	for _, c := range g.Carried {
@@ -741,8 +754,8 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 			}
 			g.Waits[m] = w
 		}
-		// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the
+		// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
-		// three readings' first two minutes after the send, the controller's one after the witness window.
+		// controller's and the tool runner's after the witness window, any other module's from the send.
 		if now.Before(j.floors[m]) {
 			continue
 		}
@@ -754,12 +767,13 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 		}
 	}
 	// What the send needs is what its most demanding module needs — the most readings, then the latest
-	// floor: a send carrying the node-engine is judged by its three readings.
+	// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
-	wants, floor, said := 0, time.Time{}, ""
+	// its two minutes (novox/hq ADR 0352).
+	wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
 	for _, m := range modules {
 		r, f := j.rules[m], j.floors[m]
 		if r.passes > wants || (r.passes == wants && f.After(floor)) {
-			wants, said = r.passes, r.said
+			wants, within, said = r.passes, r.within, r.said
 		}
 		if f.After(floor) {
 			floor = f
@@ -768,14 +782,19 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 	if len(modules) == 0 {
 		// A send judged on no machine of its modules (its first machine not running them) has no class to
 		// be judged by: it is judged as the core, never passed at once without a reading.
-		wants, floor = gatePasses, g.Since.Add(gateSettle)
+		wants, floor, within = gatePasses, *g.Since, gateWithin
-		said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle)
+		said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
+	}
+	// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
+	heldLongEnough := func(within time.Duration) bool {
+		return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
 	}
 	g.Wants = wants
 	// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
 	// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
 	passedAlone := func(m string) bool {
-		return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes
+		r := j.rules[m]
+		return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
 	}
 	// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
 	fail := func(why string) {
@@ -792,6 +811,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 	pastBound := now.Sub(*g.Since) > gateBound
 	switch {
 	case worst == healthBroken:
+		g.Faulted = true
 		g.Read(now, false, g.BrokenWhy)
 		var judging []string
 		for _, m := range modules {
@@ -810,9 +830,11 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 	case worst == healthWaiting:
 		// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
 		// the provider's own condition says what is wrong (ADR 0240 rule 5).
+		g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		g.Read(now, false, why)
 	case worst == healthNotYet:
+		g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		g.Read(now, false, why)
 		if pastBound {
@@ -831,7 +853,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
 		g.Read(now, true, "")
-		if g.Passes >= wants {
+		if g.Passes >= wants || heldLongEnough(within) {
 			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
 				now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
 		}

cmd/mesh-controller/gate_followup_test.go

@@ -21,15 +21,15 @@ import (
 // wait, and a provider waiting for a login says who waits on it.
 
 // withGateBounds sets the gate's bounds for one test.
-func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
+func withGateBounds(t *testing.T, within, every, bound time.Duration) {
 	t.Helper()
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
-	gateSettle, gateEvery, gateBound = settle, every, bound
+	gateWithin, gateEvery, gateBound = within, every, bound
-	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+	t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
 }
 
 // factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
@@ -60,7 +60,7 @@ func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
 	judgeLeavesAsTheCore(t)
 	open := aMesh(t)
 	ctx := t.Context()
-	withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
+	withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second)
 	since := time.Now().Add(-time.Second)
 	base := factsOn(t, "laptop", since, healthyContainer("app"))
 	wasGather := gatherGateFacts
@@ -101,7 +101,7 @@ const broker_runtime = "node-tools"
 func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
 	open := aMesh(t)
 	ctx := t.Context()
-	withGateBounds(t, 0, time.Hour, time.Hour)
+	withGateBounds(t, time.Hour, time.Hour, time.Hour)
 	since := time.Now().Add(-time.Minute)
 	base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
 		Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
@@ -380,7 +380,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
 		f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
 		return f, nil
 	}
-	withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
+	withGateBounds(t, time.Hour, 0, 10*time.Second)
 
 	release := func() inventory.Plan {
 		t.Helper()

cmd/mesh-controller/gate_rule_test.go

@@ -10,19 +10,19 @@ import (
 	"github.com/novox/mesh-controller/internal/lease"
 )
 
-// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one
+// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
-// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver
+// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
-// keep three, the first two minutes after the send; the bus is kept as it was.
+// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
 
 // withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
 func withTheGatesTimes(t *testing.T) {
 	t.Helper()
-	wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow
+	wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
 	wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
-	gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
+	gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
 	gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
 	t.Cleanup(func() {
-		gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow
+		gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
 		gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
 	})
 }
@@ -33,7 +33,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
 		module    string
 		claims    []catalogue.Claim
 		passes    int
-		settle    time.Duration
+		within    time.Duration
 		witnessed bool
 		said      string
 	}{
@@ -46,8 +46,8 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
 		{"app", nil, 1, 0, false, "a leaf module's rule"},
 	} {
 		r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
-		if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
+		if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
-			t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle,
+			t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
 				c.witnessed, c.said)
 		}
 	}
@@ -161,38 +161,107 @@ func TestAControllerThatFailsIsStillFailed(t *testing.T) {
 	}
 }
 
-// The node-engine and the resolver keep three readings, the first at least two minutes after the send.
+// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
-func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) {
+func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
+	t.Helper()
+	g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
+	rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
+	floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
+	return g, func(after time.Duration, h health) {
+		why := ""
+		if h != healthGood {
+			why = "the node-engine reports another build"
+		}
+		tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
+			rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
+	}
+}
+
+// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
+// minutes (novox/hq ADR 0352).
+func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
 	withTheGatesTimes(t)
 	for _, module := range []string{"mesh-host", "dnsmasq"} {
-		sent := time.Now()
+		g, judge := threeReadings(t, module, time.Now())
-		o := aGateOn(module, "ace", sent)
+		judge(15*time.Second, healthGood)
-		rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
+		judge(55*time.Second, healthGood)
-		floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
+		if g.Verdict != "" || g.Passes != 2 {
-		judge := func(after time.Duration) {
+			t.Fatalf("%s: after two readings %+v; want still judging", module, g)
-			tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood},
-				worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}},
-				sent.Add(after))
 		}
-		for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} {
+		judge(95*time.Second, healthGood)
-			judge(after)
+		if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
-		}
+			!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
-		if o.g.Passes != 0 || o.g.Verdict != "" {
+			t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
-			t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g)
-		}
-		judge(2 * time.Minute)
-		judge(160 * time.Second)
-		if o.g.Verdict != "" || o.g.Passes != 2 {
-			t.Fatalf("%s: after two readings %+v; want still judging", module, o.g)
-		}
-		judge(200 * time.Second)
-		if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 ||
-			!strings.Contains(o.g.Why, "the core's rule: 3 readings") {
-			t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g)
 		}
 	}
 }
 
+// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
+// two minutes comes sooner than the spacing after the last pass.
+func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
+	withTheGatesTimes(t)
+	sent := time.Now()
+	last := sent.Add(100 * time.Second)
+	held := &inventory.PlanGate{Since: &sent, LastPass: &last}
+	if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
+		t.Fatal("the judging at two minutes is not due, or is due before")
+	}
+	held.Faulted = true
+	if dueAtTheMost(held, sent.Add(2*time.Minute)) {
+		t.Fatal("a send that was not healthy at a reading is judged at two minutes")
+	}
+
+	g, judge := threeReadings(t, "mesh-host", time.Now())
+	judge(100*time.Second, healthGood)
+	judge(2*time.Minute, healthGood)
+	if g.Verdict != inventory.GatePassed || g.Passes != 2 {
+		t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
+	}
+}
+
+// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
+func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
+	withTheGatesTimes(t)
+	g, judge := threeReadings(t, "mesh-host", time.Now())
+	judge(10*time.Second, healthNotYet)
+	judge(100*time.Second, healthGood)
+	judge(2*time.Minute, healthGood)
+	if g.Verdict != inventory.GatePassed || g.Faulted {
+		t.Fatalf("%+v; want passed at two minutes", g)
+	}
+}
+
+// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
+// never healthy fails at the bound and is put back; one found broken fails.
+func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
+	withTheGatesTimes(t)
+	g, judge := threeReadings(t, "mesh-host", time.Now())
+	judge(20*time.Second, healthNotYet)
+	judge(60*time.Second, healthGood)
+	judge(100*time.Second, healthGood)
+	judge(2*time.Minute+10*time.Second, healthGood)
+	if g.Verdict != inventory.GatePassed || g.Passes != 3 {
+		t.Fatalf("%+v; want passed only on its third healthy reading", g)
+	}
+	g, judge = threeReadings(t, "mesh-host", time.Now())
+	judge(60*time.Second, healthGood)
+	judge(100*time.Second, healthNotYet)
+	judge(2*time.Minute, healthGood)
+	if g.Verdict != "" || !g.Faulted {
+		t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
+	}
+	judge(10*time.Minute+time.Second, healthNotYet)
+	if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
+		t.Fatalf("never healthy again: %+v; want failed at the bound", g)
+	}
+	g, judge = threeReadings(t, "mesh-host", time.Now())
+	judge(60*time.Second, healthGood)
+	judge(100*time.Second, healthBroken)
+	if g.Verdict != inventory.GateFailed {
+		t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
+	}
+}
+
 // The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
 func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
 	withTheGatesTimes(t)
@@ -226,11 +295,11 @@ func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
 		rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
 		floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
 	}
-	for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} {
+	for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
 		tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
 			"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
 	}
-	if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 {
+	if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
-		t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g)
+		t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
 	}
 }

cmd/mesh-controller/gate_test.go

@@ -124,14 +124,14 @@ func aGateMesh(t *testing.T) *gateMesh {
 	}
 	t.Cleanup(func() { gatherGateFacts = wasGather })
 
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
 	// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
-	gateSettle, gateEvery = 0, time.Hour
+	gateWithin, gateEvery = time.Hour, time.Hour
-	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+	t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
 
 	built := time.Now().UTC()
 	plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
@@ -276,7 +276,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
 func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
 	g := aGateMesh(t)
 	ctx := t.Context()
-	gateSettle = time.Hour    // a core module could not pass for an hour
+	gateWithin = time.Hour    // a core module could not pass on fewer readings for an hour
 	advancePlans(ctx, g.open) // the first machine is sent the new build
 	advancePlans(ctx, g.open) // one healthy judging
 	p := g.plan(t)

cmd/mesh-controller/release_test.go

@@ -133,15 +133,15 @@ func aBacklog(t *testing.T) *backlogMesh {
 		}
 		return f, nil
 	}
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
-	gateSettle, gateEvery = 0, time.Hour
+	gateWithin, gateEvery = time.Hour, time.Hour
 	t.Cleanup(func() {
 		machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
-		gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
+		gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
 	})
 	return b
 }

cmd/mesh-controller/replay318_test.go

@@ -112,14 +112,14 @@ func TestReplay318(t *testing.T) {
 	}
 
 	wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
 	t.Cleanup(func() {
 		machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
-		gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
+		gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
 	})
 	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
 		modules, err := open.inventory.Assigned(ctx, node)
@@ -188,7 +188,7 @@ func TestReplay318(t *testing.T) {
 		return f, nil
 	}
 	// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
-	gateSettle, gateEvery, gateBound = 0, 0, 0
+	gateWithin, gateEvery, gateBound = time.Hour, 0, 0
 
 	for i := 0; i < 12; i++ {
 		advancePlans(ctx, open)

cmd/mesh-controller/replays_test.go

@@ -248,13 +248,13 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
 		return names, nil
 	}
 	t.Cleanup(func() { sendRollout = wasSend })
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
-	gateSettle, gateEvery = 0, 0
+	gateWithin, gateEvery = time.Hour, 0
-	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+	t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
 
 	built := time.Now().UTC()
 	plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",

cmd/mesh-controller/settings_walk_test.go

@@ -271,16 +271,16 @@ func aSettledMesh(t *testing.T) *walkMesh {
 		}
 		return f, nil
 	}
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
-	gateSettle, gateEvery = 0, time.Hour
+	gateWithin, gateEvery = time.Hour, time.Hour
 	t.Cleanup(func() {
 		machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
 		settingsDataMoves = wasData
-		gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
+		gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
 	})
 	return w
 }

cmd/mesh-controller/tier_send_test.go

@@ -130,14 +130,14 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh {
 	}
 	t.Cleanup(func() { gatherGateFacts = wasGather })
 
-	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
 	// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
 	wasWindow := gateWitnessWindow
 	gateWitnessWindow = 0
 	t.Cleanup(func() { gateWitnessWindow = wasWindow })
 	// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
-	gateSettle, gateEvery = 0, time.Hour
+	gateWithin, gateEvery = time.Hour, time.Hour
-	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+	t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
 
 	built := time.Now().UTC()
 	states := map[string]*inventory.PlanModule{}
@@ -409,7 +409,7 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
 				}
 				return f, err
 			}
-			gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second
+			gateEvery, gateWithin, gateBound = 0, time.Hour, 10*time.Second
 			for i := 0; i < 20 && len(seen) < 2; i++ {
 				advancePlans(ctx, tm.open)
 			}

internal/inventory/plans.go

@@ -330,11 +330,15 @@ type PlanGate struct {
 	// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
 	// does not resets them.
 	Passes int `json:"passes,omitempty"`
-	// Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any
+	// Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two
-	// other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate
+	// minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR
+	// 0282 decision 4 and ADR 0304 decision 3). Zero in a gate
 	// judged before it was said: three.
 	Wants    int        `json:"wants,omitempty"`
 	LastPass *time.Time `json:"last_pass,omitempty"`
+	// Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading
+	// rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352).
+	Faulted bool `json:"faulted,omitempty"`
 	// Last is what the newest judging found wanting, while it still may pass.
 	Last string `json:"last,omitempty"`
 	// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.