diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 8d3bab69..b4dd98e2 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -27,8 +27,10 @@ import ( // **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by -// the component's health — the core's health definitions, or a module's own — passing three times over -// at least two minutes, within ten minutes of the apply. Only then are the rest sent. +// the component's health — the core's health definitions, or a module's own — within ten minutes of the +// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the +// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then +// are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // at its gate — registration refuses it and nothing sends it again on its own — the module's registered @@ -43,11 +45,38 @@ var ( gateSettle = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least. - gatePasses = 3 - gateEvery = 40 * time.Second + // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the + // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every + // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304 + // decision 3 changed it, issue 524). + gatePasses = 3 + gateLeafPasses = 1 + gateLeafSettles = false + gateEvery = 40 * time.Second ) +// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's +// own path, or one holding the mesh's resolver. Every other module is a leaf. +func coreModule(module string, m catalogue.Manifest) bool { + if _, own := onTheControllersPath[module]; own { + return true + } + for _, c := range m.Claims { + if c.Name == resolverSeat { + return true + } + } + return coreComponent(module) != "" +} + +// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first. +func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) { + if coreModule(module, m) { + return gatePasses, true + } + return gateLeafPasses, gateLeafSettles +} + // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions // are raised by a plan as it judges, and kept or cleared by the probe on every run. const gateProbe = "DG" @@ -608,8 +637,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs } } settled := now.Sub(*g.Since) >= gateSettle + // What the send needs is what its most demanding module needs: a send carrying a core module is judged + // as the core. + wants, settles := 0, false + for _, m := range modules { + p, s := gateWants(m, shelf[m]) + wants, settles = max(wants, p), settles || s + } + if len(modules) == 0 { + // A send judged on no machine of its modules (its first machine not running them) has no class to + // be judged by: it is judged as the core, never passed at once without a reading. + wants, settles = gatePasses, true + } + g.Wants = wants + // A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after + // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses + p, s := gateWants(m, shelf[m]) + return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -657,7 +702,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= gatePasses && settled { + if g.Passes >= wants && (settled || !settles) { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) } @@ -1340,9 +1385,9 @@ func gateLine(g *inventory.PlanGate) string { if g.Why != "" { line += ": " + g.Why } else if g.Last != "" { - line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last) + line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last) } else if g.Verdict == "" { - line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses) + line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g)) } if g.Verdict == "" && len(g.Waits) > 0 { line += waitsSaid(g.Waits) @@ -1385,3 +1430,11 @@ func witnessObservation(node string, r lease.Rollback) conditions.Observation { Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component, short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)} } + +// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said. +func gateWanted(g *inventory.PlanGate) int { + if g.Wants > 0 { + return g.Wants + } + return gatePasses +} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 54022686..35929eb5 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -53,6 +53,7 @@ func healthyContainer(module string) inventory.ResourceHealth { // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, // counted over the gate's spacing — before it says its own. The broken one alone is put back. func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { + judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) @@ -284,6 +285,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { // tools are marked and put back in the very judging that found them broken — their registered build is the // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { + judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 8fe19c58..ced2ccce 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -153,6 +153,7 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan { // registered and sent to it again — and never reaches the second machine; it is marked, said as a // condition and an event, never registered or rolled back again. func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -255,7 +256,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { t.Fatalf("sent %v", g.sent) } gate := p.Modules["app"].Gate - if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses { + if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) } if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { @@ -266,8 +267,52 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { } } +// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the +// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). +func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateSettle = time.Hour // a core module could not pass for an hour + advancePlans(ctx, g.open) // the first machine is sent the new build + advancePlans(ctx, g.open) // one healthy judging + p := g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { + t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) + } + if !strings.Contains(gate.Why, "healthy 1 times") { + t.Errorf("the verdict says %q", gate.Why) + } +} + +// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three +// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3). +func TestAModuleIsJudgedByItsClass(t *testing.T) { + for _, c := range []struct { + module string + claims []catalogue.Claim + passes int + settles bool + }{ + {"app", nil, 1, false}, + {"nzbget", nil, 1, false}, + {"mesh-controller", nil, 3, true}, + {"mesh-host", nil, 3, true}, + {"node-tools", nil, 3, true}, + {"nats", nil, 3, true}, + {catalogue.DeliverySeat, nil, 3, true}, + {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true}, + } { + p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) + if p != c.passes || s != c.settles { + t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles) + } + } +} + // A first machine that never becomes healthy fails its gate at the bound, and is put back. func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() advancePlans(ctx, g.open) @@ -593,3 +638,13 @@ func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) } } + +// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle +// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still +// open after a healthy reading. +func judgeLeavesAsTheCore(t *testing.T) { + t.Helper() + wasPasses, wasSettles := gateLeafPasses, gateLeafSettles + gateLeafPasses, gateLeafSettles = gatePasses, true + t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles }) +} diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go index 10afa237..0e48b13b 100644 --- a/cmd/mesh-controller/issue352_test.go +++ b/cmd/mesh-controller/issue352_test.go @@ -25,6 +25,7 @@ import ( // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // to another build supersedes the judging: no verdict, nothing put back. func TestReplay352(t *testing.T) { + judgeLeavesAsTheCore(t) t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) } @@ -66,6 +67,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { // A plan's own first send waits while a release judges the same module on that machine with another build. func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { + judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() advancePlans(ctx, b.open) // the release judges app c2 on anchor @@ -80,6 +82,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { // A merge plan's judging is superseded the same way: another send moved its module on the first machine. func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 11c4c828..0560cdd8 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -160,6 +160,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan { // The backlog goes out one machine at a time: the first judged before the second is sent, each build's // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { + judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go index 1b0c4d33..2ec3e0f3 100644 --- a/cmd/mesh-controller/replay335_test.go +++ b/cmd/mesh-controller/replay335_test.go @@ -77,6 +77,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // carried module's step said the first machine never applied it, and the passed build was put back. func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { + judgeLeavesAsTheCore(t) tm := aTierMesh(t, "m01", "m02") ctx := t.Context() inv := tm.open.inventory diff --git a/cmd/mesh-controller/same_source_test.go b/cmd/mesh-controller/same_source_test.go index be2dc8c2..01abd454 100644 --- a/cmd/mesh-controller/same_source_test.go +++ b/cmd/mesh-controller/same_source_test.go @@ -162,6 +162,7 @@ func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) { // A plan's build made from the source every machine running the module runs is not sent and not // judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before. func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) { + judgeLeavesAsTheCore(t) for _, tc := range []struct { name string source string diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index 34be200d..c91a8fc0 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -167,6 +167,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) { // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { + judgeLeavesAsTheCore(t) var modules []string for i := 1; i <= 12; i++ { modules = append(modules, fmt.Sprintf("m%02d", i)) @@ -268,6 +269,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { // A machine-level condition about anything else holds the machine back as a whole: everything the send // moved there fails together, once, with that reason — not one module for it. func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { + judgeLeavesAsTheCore(t) tm := aTierMesh(t, "app1", "app2", "app3") ctx := t.Context() inv := tm.open.inventory @@ -330,6 +332,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) { // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never // judged (the incident's "NOT put back: no build kept"). func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { + judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -384,6 +387,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging // the other module to its own pass, and only then fails. func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { t.Run("gate kept on "+order[0], func(t *testing.T) { tm := aTierMesh(t, order...) @@ -435,3 +439,38 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { }) } } + +// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node +// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three +// readings. +func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { + tm := aTierMesh(t, broker.RuntimeModule, "app1") + ctx := t.Context() + // One reading counted per advance, as every test with a gate left open does: a gate judged with no time + // between readings would count a pass at every look and never let an advance rest. + gateEvery, gateSettle = time.Hour, time.Hour + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p := tm.plan(t) + lead := p.Modules[broker.RuntimeModule].Gate + if lead == nil { + lead = p.Modules["app1"].Gate + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses { + t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + } + gateEvery, gateSettle = 0, 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) + } + for _, m := range []string{broker.RuntimeModule, "app1"} { + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) { + t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses) + } + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index b7d5ad24..f90999a8 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -302,7 +302,10 @@ type PlanGate struct { Sent map[string]SentDeclaration `json:"sent,omitempty"` // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. - Passes int `json:"passes,omitempty"` + Passes int `json:"passes,omitempty"` + // Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282 + // decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three. + Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"`