From 675466a8c35f0c700098161ae7d1f3856e533269 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 23:07:15 +0200 Subject: [PATCH] Hold the store tests' modules as the settle time did, and fault a dip after any healthy reading Two tier tests relied on the two-minute settle to keep a module unjudged; the witness window and the gate's spacing now hold it. A healthy reading taken before the controller's floor counted no pass, so a dip after it did not take the two-minute pass away (novox/hq ADR 0352 decision 3). --- cmd/mesh-controller/gate.go | 6 ++++-- cmd/mesh-controller/gate_rule_test.go | 28 +++++++++++++++++++++++++++ cmd/mesh-controller/tier_send_test.go | 8 +++++++- internal/inventory/plans.go | 3 +++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index de910f6e..05a2d4d0 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -830,11 +830,11 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — // the provider's own condition says what is wrong (ADR 0240 rule 5). - g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting + g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) case worst == healthNotYet: - g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting + g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) if pastBound { @@ -842,6 +842,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } case now.Before(floor): // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). + g.HealthySeen = true // The bound still holds: a module that cannot be read healthy after its floor fails at it. g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", floor.UTC().Format(time.RFC3339), said), nil @@ -850,6 +851,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. + g.HealthySeen = true g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go index 2396937f..06777685 100644 --- a/cmd/mesh-controller/gate_rule_test.go +++ b/cmd/mesh-controller/gate_rule_test.go @@ -289,6 +289,34 @@ func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { } } +// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at +// two minutes (novox/hq ADR 0352 decision 3). +func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + } + floors["mesh-host"] = sent + floors["mesh-controller"] = sent.Add(90 * time.Second) + judge := func(after time.Duration, h health) { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h}, + worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after)) + } + judge(30*time.Second, healthGood) + if g.Passes != 0 { + t.Fatalf("a reading before the controller's floor was counted: %+v", g) + } + judge(70*time.Second, healthNotYet) + judge(2*time.Minute, healthGood) + if !g.Faulted || g.Verdict != "" { + t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g) + } +} + // A send carrying the controller and the node-engine is judged by the node-engine's three readings. func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { withTheGatesTimes(t) diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index c8a21dcd..97241324 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -226,6 +226,9 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) ctx := t.Context() inv := tm.open.inventory + // The node tools are held inside their witness window, which the node-engine keeps while they settle: their + // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). + gateWitnessWindow = time.Hour advancePlans(ctx, tm.open) if len(tm.sent) != 1 { @@ -409,7 +412,10 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } return f, err } - gateEvery, gateWithin, gateBound = 0, time.Hour, 10*time.Second + // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the + // judgings that find the node tools broken, and is judged to its own pass after them (the hold the + // settle time gave before novox/hq ADR 0352 counted readings from the send). + gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second for i := 0; i < 20 && len(seen) < 2; i++ { advancePlans(ctx, tm.open) } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index a91a118f..5f058270 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -339,6 +339,9 @@ type PlanGate struct { // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). Faulted bool `json:"faulted,omitempty"` + // HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor + // counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3). + HealthySeen bool `json:"healthy_seen,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"` // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.