From c3946e5d72c2e6c57e56ba487c1909673c2fdd34 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 22:55:46 +0200 Subject: [PATCH] Pass the node-engine and resolver on their third reading, at most two minutes The operator wants these gates as fast as they can be: counting readings only after two minutes added about 80 s to every node-engine rollout for no safety. Count from the send, pass on the third healthy reading, and pass a module healthy at every reading at two minutes on what it has (novox/hq ADR 0352). --- cmd/mesh-controller/found_wait_test.go | 6 +- cmd/mesh-controller/gate.go | 78 +++++++----- cmd/mesh-controller/gate_followup_test.go | 14 +-- cmd/mesh-controller/gate_rule_test.go | 143 ++++++++++++++++------ cmd/mesh-controller/gate_test.go | 8 +- cmd/mesh-controller/release_test.go | 6 +- cmd/mesh-controller/replay318_test.go | 6 +- cmd/mesh-controller/replays_test.go | 6 +- cmd/mesh-controller/settings_walk_test.go | 6 +- cmd/mesh-controller/tier_send_test.go | 8 +- internal/inventory/plans.go | 8 +- 11 files changed, 192 insertions(+), 97 deletions(-) diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index 81c27e70..cf8eba68 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -81,13 +81,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { } return f, err } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) - gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { advancePlans(ctx, b.open) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index c140bd78..de910f6e 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -28,10 +28,11 @@ import ( // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the -// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the +// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third +// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the // node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states -// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3). +// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 +// decision 3). // Only then are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed @@ -43,12 +44,14 @@ import ( // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. var ( - // gateSettle is how long after the send a core module judged by three readings is first read, at the least. - gateSettle = 2 * time.Minute + // gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the + // gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR + // 0352). Readings gateEvery apart, the first at the apply, put three inside it. + gateWithin = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at - // the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the + // the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 // decision 4 as ADR 0304 decision 3 changed it, issue 524). gatePasses = 3 @@ -78,11 +81,12 @@ func coreModule(module string, m catalogue.Manifest) bool { } // gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the -// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's -// witness window for it. said names the rule in the gate's text. +// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it; +// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading +// (novox/hq ADR 0352). said names the rule in the gate's text. type gateRule struct { passes int - settle time.Duration + within time.Duration witnessed bool said string } @@ -105,27 +109,27 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule { case module == catalogue.DeliverySeat: return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} case coreModule(module, m): - return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+ - "the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)} + return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+ + "or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)} } r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} if gateLeafSettles { - r.settle = gateSettle - r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send", - gateLeafPasses, gateSettle) + r.within = gateWithin + r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send", + gateLeafPasses, gateWithin) } return r } -// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed -// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 -// and 4). The window opens when the node-engine starts the build, which is before the machine reports it +// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the +// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A +// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it // applied (applied, as the gate first read it) and, for the controller, before the new controller took the // lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. // A controller that starts again takes the lease again, and so moves its floor on. Pure. func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, holder *lease.Holder) time.Time { - floor := since.Add(rule.settle) + floor := since if !rule.witnessed { return floor } @@ -568,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri return judgeMoves(ctx, open, g, pairs, now) } +// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one +// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352). +func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool { + most := g.Since.Add(gateWithin) + return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most) +} + // judged is one module on one machine, as a gate judges it. type judged struct{ module, node string } @@ -578,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if g.Verdict != "" { return g.Verdict, nil } - if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery { + // A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing: + // a module healthy throughout is never held past it (novox/hq ADR 0352). + if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) { return "", nil } for _, c := range g.Carried { @@ -741,8 +754,8 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } g.Waits[m] = w } - // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the - // three readings' first two minutes after the send, the controller's one after the witness window. + // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the + // controller's and the tool runner's after the witness window, any other module's from the send. if now.Before(j.floors[m]) { continue } @@ -754,12 +767,13 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } } // What the send needs is what its most demanding module needs — the most readings, then the latest - // floor: a send carrying the node-engine is judged by its three readings. - wants, floor, said := 0, time.Time{}, "" + // floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout + // its two minutes (novox/hq ADR 0352). + wants, floor, within, said := 0, time.Time{}, time.Duration(0), "" for _, m := range modules { r, f := j.rules[m], j.floors[m] if r.passes > wants || (r.passes == wants && f.After(floor)) { - wants, said = r.passes, r.said + wants, within, said = r.passes, r.within, r.said } if f.After(floor) { floor = f @@ -768,14 +782,19 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { if len(modules) == 0 { // A send judged on no machine of its modules (its first machine not running them) has no class to // be judged by: it is judged as the core, never passed at once without a reading. - wants, floor = gatePasses, g.Since.Add(gateSettle) - said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle) + wants, floor, within = gatePasses, *g.Since, gateWithin + said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin) + } + // heldLongEnough is a rule's most: within after the send, healthy at every reading since it. + heldLongEnough := func(within time.Duration) bool { + return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within)) } g.Wants = wants // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes + r := j.rules[m] + return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within))) } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -792,6 +811,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { pastBound := now.Sub(*g.Since) > gateBound switch { case worst == healthBroken: + g.Faulted = true g.Read(now, false, g.BrokenWhy) var judging []string for _, m := range modules { @@ -810,9 +830,11 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — // the provider's own condition says what is wrong (ADR 0240 rule 5). + g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) case worst == healthNotYet: + g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) if pastBound { @@ -831,7 +853,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= wants { + if g.Passes >= wants || heldLongEnough(within) { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) } diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 6cac4ffc..b8377d77 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -21,15 +21,15 @@ import ( // wait, and a provider waiting for a login says who waits on it. // withGateBounds sets the gate's bounds for one test. -func withGateBounds(t *testing.T, settle, every, bound time.Duration) { +func withGateBounds(t *testing.T, within, every, bound time.Duration) { t.Helper() - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery, gateBound = settle, every, bound - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = within, every, bound + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) } // factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says @@ -60,7 +60,7 @@ func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() - withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) + withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second) since := time.Now().Add(-time.Second) base := factsOn(t, "laptop", since, healthyContainer("app")) wasGather := gatherGateFacts @@ -101,7 +101,7 @@ const broker_runtime = "node-tools" func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { open := aMesh(t) ctx := t.Context() - withGateBounds(t, 0, time.Hour, time.Hour) + withGateBounds(t, time.Hour, time.Hour, time.Hour) since := time.Now().Add(-time.Minute) base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) @@ -380,7 +380,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}} return f, nil } - withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second) + withGateBounds(t, time.Hour, 0, 10*time.Second) release := func() inventory.Plan { t.Helper() diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go index 3d23012f..79e7c491 100644 --- a/cmd/mesh-controller/gate_rule_test.go +++ b/cmd/mesh-controller/gate_rule_test.go @@ -10,19 +10,19 @@ import ( "github.com/novox/mesh-controller/internal/lease" ) -// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one -// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver -// keep three, the first two minutes after the send; the bus is kept as it was. +// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass +// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's +// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so. // withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. func withTheGatesTimes(t *testing.T) { t.Helper() - wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow + wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles - gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second + gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false t.Cleanup(func() { - gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow + gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles }) } @@ -33,7 +33,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) { module string claims []catalogue.Claim passes int - settle time.Duration + within time.Duration witnessed bool said string }{ @@ -46,8 +46,8 @@ func TestEachModuleHasItsGateRule(t *testing.T) { {"app", nil, 1, 0, false, "a leaf module's rule"}, } { r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { - t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle, + if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { + t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within, c.witnessed, c.said) } } @@ -161,38 +161,107 @@ func TestAControllerThatFailsIsStillFailed(t *testing.T) { } } -// The node-engine and the resolver keep three readings, the first at least two minutes after the send. -func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) { +// threeReadings judges a module under the three-reading rule on one machine, the send at sent. +func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) { + t.Helper() + g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent} + rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) + floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) + return g, func(after time.Duration, h health) { + why := "" + if h != healthGood { + why = "the node-engine reports another build" + } + tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why, + rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after)) + } +} + +// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two +// minutes (novox/hq ADR 0352). +func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) { withTheGatesTimes(t) for _, module := range []string{"mesh-host", "dnsmasq"} { - sent := time.Now() - o := aGateOn(module, "ace", sent) - rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) - floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) - judge := func(after time.Duration) { - tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood}, - worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, - sent.Add(after)) + g, judge := threeReadings(t, module, time.Now()) + judge(15*time.Second, healthGood) + judge(55*time.Second, healthGood) + if g.Verdict != "" || g.Passes != 2 { + t.Fatalf("%s: after two readings %+v; want still judging", module, g) } - for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} { - judge(after) - } - if o.g.Passes != 0 || o.g.Verdict != "" { - t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g) - } - judge(2 * time.Minute) - judge(160 * time.Second) - if o.g.Verdict != "" || o.g.Passes != 2 { - t.Fatalf("%s: after two readings %+v; want still judging", module, o.g) - } - judge(200 * time.Second) - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 || - !strings.Contains(o.g.Why, "the core's rule: 3 readings") { - t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g) + judge(95*time.Second, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 || + !strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") { + t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g) } } } +// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at +// two minutes comes sooner than the spacing after the last pass. +func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + last := sent.Add(100 * time.Second) + held := &inventory.PlanGate{Since: &sent, LastPass: &last} + if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) { + t.Fatal("the judging at two minutes is not due, or is due before") + } + held.Faulted = true + if dueAtTheMost(held, sent.Add(2*time.Minute)) { + t.Fatal("a send that was not healthy at a reading is judged at two minutes") + } + + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(100*time.Second, healthGood) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 2 { + t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) + } +} + +// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. +func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(10*time.Second, healthNotYet) + judge(100*time.Second, healthGood) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Faulted { + t.Fatalf("%+v; want passed at two minutes", g) + } +} + +// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one +// never healthy fails at the bound and is put back; one found broken fails. +func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(20*time.Second, healthNotYet) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthGood) + judge(2*time.Minute+10*time.Second, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 3 { + t.Fatalf("%+v; want passed only on its third healthy reading", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthNotYet) + judge(2*time.Minute, healthGood) + if g.Verdict != "" || !g.Faulted { + t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g) + } + judge(10*time.Minute+time.Second, healthNotYet) + if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") { + t.Fatalf("never healthy again: %+v; want failed at the bound", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthBroken) + if g.Verdict != inventory.GateFailed { + t.Fatalf("a broken reading: %+v; want failed, to be put back", g) + } +} + // The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { withTheGatesTimes(t) @@ -226,11 +295,11 @@ func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) } - for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} { + for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} { tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) } - if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 { - t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g) + if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 { + t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g) } } diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 06294a82..d8062644 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -124,14 +124,14 @@ func aGateMesh(t *testing.T) *gateMesh { } t.Cleanup(func() { gatherGateFacts = wasGather }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateSettle, gateEvery = 0, time.Hour - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", @@ -276,7 +276,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { g := aGateMesh(t) ctx := t.Context() - gateSettle = time.Hour // a core module could not pass for an hour + gateWithin = time.Hour // a core module could not pass on fewer readings for an hour advancePlans(ctx, g.open) // the first machine is sent the new build advancePlans(ctx, g.open) // one healthy judging p := g.plan(t) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 263a9f01..35b7d35c 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -133,15 +133,15 @@ func aBacklog(t *testing.T) *backlogMesh { } return f, nil } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery = 0, time.Hour + gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return b } diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 4fa3b172..53fb53ca 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -112,14 +112,14 @@ func TestReplay318(t *testing.T) { } wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) @@ -188,7 +188,7 @@ func TestReplay318(t *testing.T) { return f, nil } // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. - gateSettle, gateEvery, gateBound = 0, 0, 0 + gateWithin, gateEvery, gateBound = time.Hour, 0, 0 for i := 0; i < 12; i++ { advancePlans(ctx, open) diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 5a107b23..2798542d 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -248,13 +248,13 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { return names, nil } t.Cleanup(func() { sendRollout = wasSend }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery = 0, 0 - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, 0 + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index bf3ad9a4..45df338a 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -271,16 +271,16 @@ func aSettledMesh(t *testing.T) *walkMesh { } return f, nil } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery = 0, time.Hour + gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather settingsDataMoves = wasData - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return w } diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index df003116..c8a21dcd 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -130,14 +130,14 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh { } t.Cleanup(func() { gatherGateFacts = wasGather }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateSettle, gateEvery = 0, time.Hour - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() states := map[string]*inventory.PlanModule{} @@ -409,7 +409,7 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } return f, err } - gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second + gateEvery, gateWithin, gateBound = 0, time.Hour, 10*time.Second for i := 0; i < 20 && len(seen) < 2; i++ { advancePlans(ctx, tm.open) } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index abc02bf6..a91a118f 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -330,11 +330,15 @@ type PlanGate struct { // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any - // other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate + // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two + // minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR + // 0282 decision 4 and ADR 0304 decision 3). Zero in a gate // judged before it was said: three. Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` + // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading + // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). + Faulted bool `json:"faulted,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"` // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.