Pass the controller's first-node gate on one reading after the witness window

The controller runs on one machine, so three readings over two minutes guarded no
other machine and held every walk behind it. Judge each module by its own rule:
one reading for the controller and the tool runner once the node-engine's witness
window has ended, one for mesh-delivery, three for the node-engine and the
resolver with the first two minutes after the send (novox/hq ADR 0351). The
gate's verdict names the rule it applied.

cmd/mesh-controller/gate.go

@@ -28,9 +28,11 @@ import (
 // that machine reported it applied. A build that applies and then does nothing, crashes, serves no
 // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
 // the component's health — the core's health definitions, or a module's own — within ten minutes of the
-// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
-// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
-// are the rest sent.
+// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the
+// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the
+// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
+// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3).
+// Only then are the rest sent.
 //
 // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
 // at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -41,22 +43,28 @@ import (
 
 // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
 var (
-	// gateSettle is how long the first machine must stay healthy, at the least.
+	// gateSettle is how long after the send a core module judged by three readings is first read, at the least.
 	gateSettle = 2 * time.Minute
 	// gateBound is how long after the apply the build may take to become healthy.
 	gateBound = 10 * time.Minute
-	// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
-	// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
-	// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
-	// decision 3 changed it, issue 524).
+	// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
+	// the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the
+	// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
+	// decision 4 as ADR 0304 decision 3 changed it, issue 524).
 	gatePasses      = 3
 	gateLeafPasses  = 1
 	gateLeafSettles = false
 	gateEvery       = 40 * time.Second
+	// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
+	// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
+	// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
+	// 0351 decisions 1 and 4).
+	gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
 )
 
 // coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
-// own path, or one holding the mesh's resolver. Every other module is a leaf.
+// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
+// delivery budget; how many readings its gate takes is gateRuleOf's.
 func coreModule(module string, m catalogue.Manifest) bool {
 	if _, own := onTheControllersPath[module]; own {
 		return true
@@ -69,12 +77,71 @@ func coreModule(module string, m catalogue.Manifest) bool {
 	return coreComponent(module) != ""
 }
 
-// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
-func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
-	if coreModule(module, m) {
-		return gatePasses, true
+// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
+// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's
+// witness window for it. said names the rule in the gate's text.
+type gateRule struct {
+	passes    int
+	settle    time.Duration
+	witnessed bool
+	said      string
+}
+
+// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
+// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
+// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
+// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
+// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
+// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
+// decision 3).
+func gateRuleOf(module string, m catalogue.Manifest) gateRule {
+	switch {
+	case coreComponent(module) == lease.ComponentController:
+		return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
+			"after the node-engine's witness window (novox/hq ADR 0351)"}
+	case coreComponent(module) == lease.ComponentNodeTools:
+		return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
+			"after the node-engine's witness window (novox/hq ADR 0351)"}
+	case module == catalogue.DeliverySeat:
+		return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
+	case coreModule(module, m):
+		return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+
+			"the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)}
 	}
-	return gateLeafPasses, gateLeafSettles
+	r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
+	if gateLeafSettles {
+		r.settle = gateSettle
+		r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send",
+			gateLeafPasses, gateSettle)
+	}
+	return r
+}
+
+// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed
+// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1
+// and 4). The window opens when the node-engine starts the build, which is before the machine reports it
+// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
+// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
+// A controller that starts again takes the lease again, and so moves its floor on. Pure.
+func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
+	holder *lease.Holder) time.Time {
+	floor := since.Add(rule.settle)
+	if !rule.witnessed {
+		return floor
+	}
+	for _, n := range machines {
+		start := since
+		if at, read := applied[n]; read && at.After(start) {
+			start = at
+		}
+		if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
+			start = holder.Taken
+		}
+		if end := start.Add(gateWitnessWindow); end.After(floor) {
+			floor = end
+		}
+	}
+	return floor
 }
 
 // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
@@ -608,6 +675,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 			why = said
 		}
 	}
+	// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
+	// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
+	if g.Applied == nil {
+		g.Applied = map[string]time.Time{}
+	}
+	for node := range byMachine {
+		r, said := facts.reports[node]
+		if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
+			facts.reportedOn(node, r) {
+			g.Applied[node] = *r.At
+		}
+	}
+	rules, floors := map[string]gateRule{}, map[string]time.Time{}
+	for _, m := range modules {
+		var on []string
+		for _, j := range pairs {
+			if j.module == m {
+				on = append(on, j.node)
+			}
+		}
+		rules[m] = gateRuleOf(m, shelf[m])
+		floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
+	}
+	tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
+		rules: rules, floors: floors}, now)
+	return g.Verdict, nil
+}
+
+// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
+// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
+// rule and the earliest its reading counts.
+type gateJudging struct {
+	modules []string
+	reading map[string]health
+	waits   map[string]string
+	worst   health
+	why     string
+	rules   map[string]gateRule
+	floors  map[string]time.Time
+}
+
+// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
+// for the gate it writes.
+func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
+	modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
 	if g.Healthy == nil {
 		g.Healthy = map[string]int{}
 	}
@@ -623,38 +735,47 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 			delete(g.HealthyAt, m)
 			continue
 		}
-		// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
-		// often while another module beside it is not yet healthy counts no faster.
-		if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
-			g.Healthy[m]++
-			g.HealthyAt[m] = now
-		}
 		if w := waits[m]; w != "" {
 			if g.Waits == nil {
 				g.Waits = map[string]string{}
 			}
 			g.Waits[m] = w
 		}
+		// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the
+		// three readings' first two minutes after the send, the controller's one after the witness window.
+		if now.Before(j.floors[m]) {
+			continue
+		}
+		// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
+		// often while another module beside it is not yet healthy counts no faster.
+		if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
+			g.Healthy[m]++
+			g.HealthyAt[m] = now
+		}
 	}
-	settled := now.Sub(*g.Since) >= gateSettle
-	// What the send needs is what its most demanding module needs: a send carrying a core module is judged
-	// as the core.
-	wants, settles := 0, false
+	// What the send needs is what its most demanding module needs — the most readings, then the latest
+	// floor: a send carrying the node-engine is judged by its three readings.
+	wants, floor, said := 0, time.Time{}, ""
 	for _, m := range modules {
-		p, s := gateWants(m, shelf[m])
-		wants, settles = max(wants, p), settles || s
+		r, f := j.rules[m], j.floors[m]
+		if r.passes > wants || (r.passes == wants && f.After(floor)) {
+			wants, said = r.passes, r.said
+		}
+		if f.After(floor) {
+			floor = f
+		}
 	}
 	if len(modules) == 0 {
 		// A send judged on no machine of its modules (its first machine not running them) has no class to
 		// be judged by: it is judged as the core, never passed at once without a reading.
-		wants, settles = gatePasses, true
+		wants, floor = gatePasses, g.Since.Add(gateSettle)
+		said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle)
 	}
 	g.Wants = wants
-	// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
+	// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
 	// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
 	passedAlone := func(m string) bool {
-		p, s := gateWants(m, shelf[m])
-		return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
+		return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes
 	}
 	// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
 	fail := func(why string) {
@@ -697,17 +818,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		if pastBound {
 			fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
 		}
+	case now.Before(floor):
+		// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
+		// The bound still holds: a module that cannot be read healthy after its floor fails at it.
+		g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
+			floor.UTC().Format(time.RFC3339), said), nil
+		if pastBound {
+			fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
+		}
 	default:
 		// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
 		g.Read(now, true, "")
-		if g.Passes >= wants && (settled || !settles) {
-			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
-				now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
+		if g.Passes >= wants {
+			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
+				now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
 		}
 	}
-	return g.Verdict, nil
 }
 
 // judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each