Merge pull request 'Judge a leaf module on one healthy reading, a core module on three (hq issue 524)' (#240) from fix/524-judgement-by-class into main

cmd/mesh-controller/gate.go

@@ -27,8 +27,10 @@ import (
 // **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
 // that machine reported it applied. A build that applies and then does nothing, crashes, serves no
 // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
-// the component's health — the core's health definitions, or a module's own — passing three times over
+// the component's health — the core's health definitions, or a module's own — within ten minutes of the
-// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
+// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
+// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
+// are the rest sent.
 //
 // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
 // at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -43,11 +45,38 @@ var (
 	gateSettle = 2 * time.Minute
 	// gateBound is how long after the apply the build may take to become healthy.
 	gateBound = 10 * time.Minute
-	// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
+	// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
-	gatePasses = 3
+	// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
-	gateEvery  = 40 * time.Second
+	// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
+	// decision 3 changed it, issue 524).
+	gatePasses      = 3
+	gateLeafPasses  = 1
+	gateLeafSettles = false
+	gateEvery       = 40 * time.Second
 )
 
+// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
+// own path, or one holding the mesh's resolver. Every other module is a leaf.
+func coreModule(module string, m catalogue.Manifest) bool {
+	if _, own := onTheControllersPath[module]; own {
+		return true
+	}
+	for _, c := range m.Claims {
+		if c.Name == resolverSeat {
+			return true
+		}
+	}
+	return coreComponent(module) != ""
+}
+
+// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
+func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
+	if coreModule(module, m) {
+		return gatePasses, true
+	}
+	return gateLeafPasses, gateLeafSettles
+}
+
 // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
 // are raised by a plan as it judges, and kept or cleared by the probe on every run.
 const gateProbe = "DG"
@@ -608,8 +637,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		}
 	}
 	settled := now.Sub(*g.Since) >= gateSettle
+	// What the send needs is what its most demanding module needs: a send carrying a core module is judged
+	// as the core.
+	wants, settles := 0, false
+	for _, m := range modules {
+		p, s := gateWants(m, shelf[m])
+		wants, settles = max(wants, p), settles || s
+	}
+	if len(modules) == 0 {
+		// A send judged on no machine of its modules (its first machine not running them) has no class to
+		// be judged by: it is judged as the core, never passed at once without a reading.
+		wants, settles = gatePasses, true
+	}
+	g.Wants = wants
+	// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
+	// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
 	passedAlone := func(m string) bool {
-		return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
+		p, s := gateWants(m, shelf[m])
+		return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
 	}
 	// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
 	fail := func(why string) {
@@ -657,7 +702,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
 		g.Read(now, true, "")
-		if g.Passes >= gatePasses && settled {
+		if g.Passes >= wants && (settled || !settles) {
 			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
 				now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
 		}
@@ -1340,9 +1385,9 @@ func gateLine(g *inventory.PlanGate) string {
 	if g.Why != "" {
 		line += ": " + g.Why
 	} else if g.Last != "" {
-		line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
+		line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last)
 	} else if g.Verdict == "" {
-		line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
+		line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g))
 	}
 	if g.Verdict == "" && len(g.Waits) > 0 {
 		line += waitsSaid(g.Waits)
@@ -1385,3 +1430,11 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation {
 		Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
 			short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
 }
+
+// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said.
+func gateWanted(g *inventory.PlanGate) int {
+	if g.Wants > 0 {
+		return g.Wants
+	}
+	return gatePasses
+}

cmd/mesh-controller/gate_followup_test.go

@@ -53,6 +53,7 @@ func healthyContainer(module string) inventory.ResourceHealth {
 // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
 // counted over the gate's spacing — before it says its own. The broken one alone is put back.
 func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	open := aMesh(t)
 	ctx := t.Context()
 	withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
@@ -284,6 +285,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
 // tools are marked and put back in the very judging that found them broken — their registered build is the
 // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
 func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	open := aMesh(t)
 	ctx := t.Context()
 	inv := open.inventory

cmd/mesh-controller/gate_test.go

@@ -153,6 +153,7 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan {
 // registered and sent to it again — and never reaches the second machine; it is marked, said as a
 // condition and an event, never registered or rolled back again.
 func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	g := aGateMesh(t)
 	ctx := t.Context()
 	inv := g.open.inventory
@@ -255,7 +256,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
 		t.Fatalf("sent %v", g.sent)
 	}
 	gate := p.Modules["app"].Gate
-	if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
+	if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
 		t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
 	}
 	if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
@@ -266,8 +267,52 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
 	}
 }
 
+// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
+// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
+func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
+	g := aGateMesh(t)
+	ctx := t.Context()
+	gateSettle = time.Hour    // a core module could not pass for an hour
+	advancePlans(ctx, g.open) // the first machine is sent the new build
+	advancePlans(ctx, g.open) // one healthy judging
+	p := g.plan(t)
+	gate := p.Modules["app"].Gate
+	if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
+		t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
+	}
+	if !strings.Contains(gate.Why, "healthy 1 times") {
+		t.Errorf("the verdict says %q", gate.Why)
+	}
+}
+
+// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
+// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
+func TestAModuleIsJudgedByItsClass(t *testing.T) {
+	for _, c := range []struct {
+		module  string
+		claims  []catalogue.Claim
+		passes  int
+		settles bool
+	}{
+		{"app", nil, 1, false},
+		{"nzbget", nil, 1, false},
+		{"mesh-controller", nil, 3, true},
+		{"mesh-host", nil, 3, true},
+		{"node-tools", nil, 3, true},
+		{"nats", nil, 3, true},
+		{catalogue.DeliverySeat, nil, 3, true},
+		{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
+	} {
+		p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
+		if p != c.passes || s != c.settles {
+			t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
+		}
+	}
+}
+
 // A first machine that never becomes healthy fails its gate at the bound, and is put back.
 func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	g := aGateMesh(t)
 	ctx := t.Context()
 	advancePlans(ctx, g.open)
@@ -593,3 +638,13 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
 		t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
 	}
 }
+
+// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
+// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
+// open after a healthy reading.
+func judgeLeavesAsTheCore(t *testing.T) {
+	t.Helper()
+	wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
+	gateLeafPasses, gateLeafSettles = gatePasses, true
+	t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
+}

cmd/mesh-controller/issue352_test.go

@@ -25,6 +25,7 @@ import (
 // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
 // to another build supersedes the judging: no verdict, nothing put back.
 func TestReplay352(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
 	t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
 }
@@ -66,6 +67,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
 
 // A plan's own first send waits while a release judges the same module on that machine with another build.
 func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	b := aBacklog(t)
 	ctx := t.Context()
 	advancePlans(ctx, b.open) // the release judges app c2 on anchor
@@ -80,6 +82,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
 
 // A merge plan's judging is superseded the same way: another send moved its module on the first machine.
 func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	g := aGateMesh(t)
 	ctx := t.Context()
 	inv := g.open.inventory

cmd/mesh-controller/release_test.go

@@ -160,6 +160,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
 // The backlog goes out one machine at a time: the first judged before the second is sent, each build's
 // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
 func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	b := aBacklog(t)
 	ctx := t.Context()
 	inv := b.open.inventory

cmd/mesh-controller/replay335_test.go

@@ -77,6 +77,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
 // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
 // carried module's step said the first machine never applied it, and the passed build was put back.
 func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	tm := aTierMesh(t, "m01", "m02")
 	ctx := t.Context()
 	inv := tm.open.inventory

cmd/mesh-controller/same_source_test.go

@@ -162,6 +162,7 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
 // A plan's build made from the source every machine running the module runs is not sent and not
 // judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
 func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	for _, tc := range []struct {
 		name   string
 		source string

cmd/mesh-controller/tier_send_test.go

@@ -167,6 +167,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
 // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
 // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
 func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	var modules []string
 	for i := 1; i <= 12; i++ {
 		modules = append(modules, fmt.Sprintf("m%02d", i))
@@ -268,6 +269,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
 // A machine-level condition about anything else holds the machine back as a whole: everything the send
 // moved there fails together, once, with that reason — not one module for it.
 func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	tm := aTierMesh(t, "app1", "app2", "app3")
 	ctx := t.Context()
 	inv := tm.open.inventory
@@ -330,6 +332,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
 // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
 // judged (the incident's "NOT put back: no build kept").
 func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	g := aGateMesh(t)
 	ctx := t.Context()
 	inv := g.open.inventory
@@ -384,6 +387,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
 // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
 // the other module to its own pass, and only then fails.
 func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
+	judgeLeavesAsTheCore(t)
 	for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
 		t.Run("gate kept on "+order[0], func(t *testing.T) {
 			tm := aTierMesh(t, order...)
@@ -435,3 +439,38 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
 		})
 	}
 }
+
+// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
+// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
+// readings.
+func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
+	tm := aTierMesh(t, broker.RuntimeModule, "app1")
+	ctx := t.Context()
+	// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
+	// between readings would count a pass at every look and never let an advance rest.
+	gateEvery, gateSettle = time.Hour, time.Hour
+	for i := 0; i < 6; i++ {
+		advancePlans(ctx, tm.open)
+	}
+	p := tm.plan(t)
+	lead := p.Modules[broker.RuntimeModule].Gate
+	if lead == nil {
+		lead = p.Modules["app1"].Gate
+	}
+	if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
+		t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
+	}
+	gateEvery, gateSettle = 0, 0
+	for i := 0; i < 6; i++ {
+		advancePlans(ctx, tm.open)
+	}
+	p = tm.plan(t)
+	if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) {
+		t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
+	}
+	for _, m := range []string{broker.RuntimeModule, "app1"} {
+		if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
+			t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
+		}
+	}
+}

internal/inventory/plans.go

@@ -302,7 +302,10 @@ type PlanGate struct {
 	Sent map[string]SentDeclaration `json:"sent,omitempty"`
 	// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
 	// does not resets them.
-	Passes   int        `json:"passes,omitempty"`
+	Passes int `json:"passes,omitempty"`
+	// Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282
+	// decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three.
+	Wants    int        `json:"wants,omitempty"`
 	LastPass *time.Time `json:"last_pass,omitempty"`
 	// Last is what the newest judging found wanting, while it still may pass.
 	Last string `json:"last,omitempty"`