From 524295926f76bee5516ac664993d8038e821b3b9 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 22:39:32 +0200 Subject: [PATCH 1/4] Pass the controller's first-node gate on one reading after the witness window The controller runs on one machine, so three readings over two minutes guarded no other machine and held every walk behind it. Judge each module by its own rule: one reading for the controller and the tool runner once the node-engine's witness window has ended, one for mesh-delivery, three for the node-engine and the resolver with the first two minutes after the send (novox/hq ADR 0351). The gate's verdict names the rule it applied. --- cmd/mesh-controller/found_wait_test.go | 4 + cmd/mesh-controller/gate.go | 196 ++++++++++++++---- cmd/mesh-controller/gate_followup_test.go | 4 + cmd/mesh-controller/gate_rule_test.go | 236 ++++++++++++++++++++++ cmd/mesh-controller/gate_test.go | 29 +-- cmd/mesh-controller/release_test.go | 4 + cmd/mesh-controller/replay318_test.go | 4 + cmd/mesh-controller/replays_test.go | 4 + cmd/mesh-controller/settings_walk_test.go | 4 + cmd/mesh-controller/tier_send_test.go | 25 ++- internal/inventory/plans.go | 9 +- 11 files changed, 448 insertions(+), 71 deletions(-) create mode 100644 cmd/mesh-controller/gate_rule_test.go diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index 3fff9be3..81c27e70 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -82,6 +82,10 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { return f, err } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index f5aa1689..c140bd78 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -28,9 +28,11 @@ import ( // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the -// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then -// are the rest sent. +// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the +// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the +// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states +// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3). +// Only then are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // at its gate — registration refuses it and nothing sends it again on its own — the module's registered @@ -41,22 +43,28 @@ import ( // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. var ( - // gateSettle is how long the first machine must stay healthy, at the least. + // gateSettle is how long after the send a core module judged by three readings is first read, at the least. gateSettle = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the - // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every - // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304 - // decision 3 changed it, issue 524). + // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at + // the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the + // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 + // decision 4 as ADR 0304 decision 3 changed it, issue 524). gatePasses = 3 gateLeafPasses = 1 gateLeafSettles = false gateEvery = 40 * time.Second + // gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its + // start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either + // counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR + // 0351 decisions 1 and 4). + gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin) ) // coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's -// own path, or one holding the mesh's resolver. Every other module is a leaf. +// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's +// delivery budget; how many readings its gate takes is gateRuleOf's. func coreModule(module string, m catalogue.Manifest) bool { if _, own := onTheControllersPath[module]; own { return true @@ -69,12 +77,71 @@ func coreModule(module string, m catalogue.Manifest) bool { return coreComponent(module) != "" } -// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first. -func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) { - if coreModule(module, m) { - return gatePasses, true +// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the +// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's +// witness window for it. said names the rule in the gate's text. +type gateRule struct { + passes int + settle time.Duration + witnessed bool + said string +} + +// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its +// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send, +// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never +// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner, +// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4); +// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304 +// decision 3). +func gateRuleOf(module string, m catalogue.Manifest) gateRule { + switch { + case coreComponent(module) == lease.ComponentController: + return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case coreComponent(module) == lease.ComponentNodeTools: + return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case module == catalogue.DeliverySeat: + return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} + case coreModule(module, m): + return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+ + "the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)} } - return gateLeafPasses, gateLeafSettles + r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} + if gateLeafSettles { + r.settle = gateSettle + r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send", + gateLeafPasses, gateSettle) + } + return r +} + +// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed +// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 +// and 4). The window opens when the node-engine starts the build, which is before the machine reports it +// applied (applied, as the gate first read it) and, for the controller, before the new controller took the +// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. +// A controller that starts again takes the lease again, and so moves its floor on. Pure. +func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, + holder *lease.Holder) time.Time { + floor := since.Add(rule.settle) + if !rule.witnessed { + return floor + } + for _, n := range machines { + start := since + if at, read := applied[n]; read && at.After(start) { + start = at + } + if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) { + start = holder.Taken + } + if end := start.Add(gateWitnessWindow); end.After(floor) { + floor = end + } + } + return floor } // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions @@ -608,6 +675,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs why = said } } + // **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read + // each machine report its send applied, and who holds the controller lease, the earliest its reading counts. + if g.Applied == nil { + g.Applied = map[string]time.Time{} + } + for node := range byMachine { + r, said := facts.reports[node] + if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied && + facts.reportedOn(node, r) { + g.Applied[node] = *r.At + } + } + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + var on []string + for _, j := range pairs { + if j.module == m { + on = append(on, j.node) + } + } + rules[m] = gateRuleOf(m, shelf[m]) + floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder) + } + tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why, + rules: rules, floors: floors}, now) + return g.Verdict, nil +} + +// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading +// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's +// rule and the earliest its reading counts. +type gateJudging struct { + modules []string + reading map[string]health + waits map[string]string + worst health + why string + rules map[string]gateRule + floors map[string]time.Time +} + +// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but +// for the gate it writes. +func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { + modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why if g.Healthy == nil { g.Healthy = map[string]int{} } @@ -623,38 +735,47 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs delete(g.HealthyAt, m) continue } - // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more - // often while another module beside it is not yet healthy counts no faster. - if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { - g.Healthy[m]++ - g.HealthyAt[m] = now - } if w := waits[m]; w != "" { if g.Waits == nil { g.Waits = map[string]string{} } g.Waits[m] = w } + // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the + // three readings' first two minutes after the send, the controller's one after the witness window. + if now.Before(j.floors[m]) { + continue + } + // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more + // often while another module beside it is not yet healthy counts no faster. + if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { + g.Healthy[m]++ + g.HealthyAt[m] = now + } } - settled := now.Sub(*g.Since) >= gateSettle - // What the send needs is what its most demanding module needs: a send carrying a core module is judged - // as the core. - wants, settles := 0, false + // What the send needs is what its most demanding module needs — the most readings, then the latest + // floor: a send carrying the node-engine is judged by its three readings. + wants, floor, said := 0, time.Time{}, "" for _, m := range modules { - p, s := gateWants(m, shelf[m]) - wants, settles = max(wants, p), settles || s + r, f := j.rules[m], j.floors[m] + if r.passes > wants || (r.passes == wants && f.After(floor)) { + wants, said = r.passes, r.said + } + if f.After(floor) { + floor = f + } } if len(modules) == 0 { // A send judged on no machine of its modules (its first machine not running them) has no class to // be judged by: it is judged as the core, never passed at once without a reading. - wants, settles = gatePasses, true + wants, floor = gatePasses, g.Since.Add(gateSettle) + said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle) } g.Wants = wants - // A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after + // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - p, s := gateWants(m, shelf[m]) - return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p + return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -697,17 +818,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if pastBound { fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) } + case now.Before(floor): + // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). + // The bound still holds: a module that cannot be read healthy after its floor fails at it. + g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", + floor.UTC().Format(time.RFC3339), said), nil + if pastBound { + fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last)) + } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= wants && (settled || !settles) { - decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, - now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) + if g.Passes >= wants { + decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, + now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) } } - return g.Verdict, nil } // judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 35929eb5..6cac4ffc 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -24,6 +24,10 @@ import ( func withGateBounds(t *testing.T, settle, every, bound time.Duration) { t.Helper() wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery, gateBound = settle, every, bound t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) } diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go new file mode 100644 index 00000000..3d23012f --- /dev/null +++ b/cmd/mesh-controller/gate_rule_test.go @@ -0,0 +1,236 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one +// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver +// keep three, the first two minutes after the send; the bus is kept as it was. + +// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. +func withTheGatesTimes(t *testing.T) { + t.Helper() + wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow + wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles + gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second + gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false + t.Cleanup(func() { + gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow + gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles + }) +} + +func TestEachModuleHasItsGateRule(t *testing.T) { + withTheGatesTimes(t) + for _, c := range []struct { + module string + claims []catalogue.Claim + passes int + settle time.Duration + witnessed bool + said string + }{ + {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, + {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, + {catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"}, + {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, + {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"app", nil, 1, 0, false, "a leaf module's rule"}, + } { + r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) + if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { + t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle, + c.witnessed, c.said) + } + } +} + +// Every module that was core stays core for its delivery budget and its walk: only its readings change. +func TestTheCoreStaysTheCore(t *testing.T) { + for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { + if !coreModule(m, catalogue.Manifest{Module: m}) { + t.Errorf("%s is no longer judged as the core for its class", m) + } + } + if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { + t.Error("the resolver's holder is no longer judged as the core for its class") + } +} + +// gateOn judges one module on one machine, healthy at every reading. +type gateOn struct { + module, node string + g *inventory.PlanGate + holder *lease.Holder +} + +func aGateOn(module, node string, since time.Time) *gateOn { + return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, + Applied: map[string]time.Time{}}} +} + +func (o *gateOn) read(at time.Time, h health, why string) { + r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module}) + floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) + tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, + worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) +} + +func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(15 * time.Second) + o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} + + // Ready, but inside the witness window, which opened before it took the lease: not counted. + for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { + o.read(sent.Add(after), healthGood, "") + if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { + t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) + } + } + o.read(sent.Add(80*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { + t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) + } + if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || + !strings.Contains(o.g.Why, "ADR 0351") { + t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) + } +} + +// A controller that starts again takes the lease again: its witness window starts again, and so does the +// gate's wait. +func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(10 * time.Second) + o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} + o.read(sent.Add(90*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) + } + o.read(sent.Add(130*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed { + t.Fatalf("not passed after the new window: %+v", o.g) + } +} + +// Without the lease's word (a controller carried by another gate), the window counts from the applied report, +// and from the send at the least. +func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + r := gateRuleOf("mesh-controller", catalogue.Manifest{}) + if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { + t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) + } + if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, + map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { + t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) + } +} + +// A controller that a witness put back, or that never becomes ready, fails its gate as before. +func TestAControllerThatFailsIsStillFailed(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") + if o.g.Verdict != inventory.GateFailed { + t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) + } + o = aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != "" { + t.Fatalf("failed before the bound: %+v", o.g) + } + o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { + t.Fatalf("never ready: %+v; want failed at the bound", o.g) + } +} + +// The node-engine and the resolver keep three readings, the first at least two minutes after the send. +func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) { + withTheGatesTimes(t) + for _, module := range []string{"mesh-host", "dnsmasq"} { + sent := time.Now() + o := aGateOn(module, "ace", sent) + rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) + floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) + judge := func(after time.Duration) { + tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood}, + worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, + sent.Add(after)) + } + for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} { + judge(after) + } + if o.g.Passes != 0 || o.g.Verdict != "" { + t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g) + } + judge(2 * time.Minute) + judge(160 * time.Second) + if o.g.Verdict != "" || o.g.Passes != 2 { + t.Fatalf("%s: after two readings %+v; want still judging", module, o.g) + } + judge(200 * time.Second) + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 || + !strings.Contains(o.g.Why, "the core's rule: 3 readings") { + t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g) + } + } +} + +// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. +func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("node-tools", "ace", sent) + o.g.Applied["ace"] = sent.Add(10 * time.Second) + o.read(sent.Add(30*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) + } + o.read(sent.Add(70*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { + t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) + } + + o = aGateOn(catalogue.DeliverySeat, "novox", sent) + o.read(sent.Add(5*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { + t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) + } +} + +// A send carrying the controller and the node-engine is judged by the node-engine's three readings. +func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) + } + for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, + "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) + } + if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 { + t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g) + } +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index ced2ccce..06294a82 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -125,6 +125,10 @@ func aGateMesh(t *testing.T) *gateMesh { t.Cleanup(func() { gatherGateFacts = wasGather }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) @@ -285,31 +289,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { } } -// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three -// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3). -func TestAModuleIsJudgedByItsClass(t *testing.T) { - for _, c := range []struct { - module string - claims []catalogue.Claim - passes int - settles bool - }{ - {"app", nil, 1, false}, - {"nzbget", nil, 1, false}, - {"mesh-controller", nil, 3, true}, - {"mesh-host", nil, 3, true}, - {"node-tools", nil, 3, true}, - {"nats", nil, 3, true}, - {catalogue.DeliverySeat, nil, 3, true}, - {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true}, - } { - p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if p != c.passes || s != c.settles { - t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles) - } - } -} - // A first machine that never becomes healthy fails its gate at the bound, and is put back. func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { judgeLeavesAsTheCore(t) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 0560cdd8..263a9f01 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -134,6 +134,10 @@ func aBacklog(t *testing.T) *backlogMesh { return f, nil } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 09e212d0..4fa3b172 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -113,6 +113,10 @@ func TestReplay318(t *testing.T) { wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 138f0180..5a107b23 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -249,6 +249,10 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { } t.Cleanup(func() { sendRollout = wasSend }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, 0 t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 48c34f2e..bf3ad9a4 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -272,6 +272,10 @@ func aSettledMesh(t *testing.T) *walkMesh { return f, nil } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index c91a8fc0..df003116 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -131,6 +131,10 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh { t.Cleanup(func() { gatherGateFacts = wasGather }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) @@ -440,15 +444,15 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } } -// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node -// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three -// readings. -func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { +// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 +// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool +// runner's rule: one reading, not before the node-engine's witness window for it has ended. +func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { tm := aTierMesh(t, broker.RuntimeModule, "app1") ctx := t.Context() // One reading counted per advance, as every test with a gate left open does: a gate judged with no time // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateSettle = time.Hour, time.Hour + gateEvery, gateWitnessWindow = time.Hour, time.Hour for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -457,10 +461,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { if lead == nil { lead = p.Modules["app1"].Gate } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses { - t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { + t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) } - gateEvery, gateSettle = 0, 0 + gateEvery, gateWitnessWindow = 0, 0 for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -469,8 +473,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) } for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) { - t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses) + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || + !strings.Contains(g.Why, "the tool runner's rule")) { + t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) } } } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index c17344a0..abc02bf6 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -323,11 +323,16 @@ type PlanGate struct { // machine's report is held against. Absent on a gate kept before it was, which reads the report // against the send made last, as before. Sent map[string]SentDeclaration `json:"sent,omitempty"` + // Applied is, per machine, the time of the first report the gate read of its send applied: the + // node-engine started the build before it, so a witness window counted from it ends no earlier than the + // node-engine's own (novox/hq ADR 0351 decisions 1 and 4). + Applied map[string]time.Time `json:"applied,omitempty"` // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282 - // decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three. + // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any + // other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate + // judged before it was said: three. Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` // Last is what the newest judging found wanting, while it still may pass. From c3946e5d72c2e6c57e56ba487c1909673c2fdd34 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 22:55:46 +0200 Subject: [PATCH 2/4] Pass the node-engine and resolver on their third reading, at most two minutes The operator wants these gates as fast as they can be: counting readings only after two minutes added about 80 s to every node-engine rollout for no safety. Count from the send, pass on the third healthy reading, and pass a module healthy at every reading at two minutes on what it has (novox/hq ADR 0352). --- cmd/mesh-controller/found_wait_test.go | 6 +- cmd/mesh-controller/gate.go | 78 +++++++----- cmd/mesh-controller/gate_followup_test.go | 14 +-- cmd/mesh-controller/gate_rule_test.go | 143 ++++++++++++++++------ cmd/mesh-controller/gate_test.go | 8 +- cmd/mesh-controller/release_test.go | 6 +- cmd/mesh-controller/replay318_test.go | 6 +- cmd/mesh-controller/replays_test.go | 6 +- cmd/mesh-controller/settings_walk_test.go | 6 +- cmd/mesh-controller/tier_send_test.go | 8 +- internal/inventory/plans.go | 8 +- 11 files changed, 192 insertions(+), 97 deletions(-) diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index 81c27e70..cf8eba68 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -81,13 +81,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { } return f, err } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) - gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { advancePlans(ctx, b.open) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index c140bd78..de910f6e 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -28,10 +28,11 @@ import ( // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the -// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the +// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third +// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the // node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states -// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3). +// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 +// decision 3). // Only then are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed @@ -43,12 +44,14 @@ import ( // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. var ( - // gateSettle is how long after the send a core module judged by three readings is first read, at the least. - gateSettle = 2 * time.Minute + // gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the + // gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR + // 0352). Readings gateEvery apart, the first at the apply, put three inside it. + gateWithin = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at - // the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the + // the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 // decision 4 as ADR 0304 decision 3 changed it, issue 524). gatePasses = 3 @@ -78,11 +81,12 @@ func coreModule(module string, m catalogue.Manifest) bool { } // gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the -// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's -// witness window for it. said names the rule in the gate's text. +// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it; +// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading +// (novox/hq ADR 0352). said names the rule in the gate's text. type gateRule struct { passes int - settle time.Duration + within time.Duration witnessed bool said string } @@ -105,27 +109,27 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule { case module == catalogue.DeliverySeat: return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} case coreModule(module, m): - return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+ - "the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)} + return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+ + "or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)} } r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} if gateLeafSettles { - r.settle = gateSettle - r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send", - gateLeafPasses, gateSettle) + r.within = gateWithin + r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send", + gateLeafPasses, gateWithin) } return r } -// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed -// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 -// and 4). The window opens when the node-engine starts the build, which is before the machine reports it +// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the +// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A +// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it // applied (applied, as the gate first read it) and, for the controller, before the new controller took the // lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. // A controller that starts again takes the lease again, and so moves its floor on. Pure. func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, holder *lease.Holder) time.Time { - floor := since.Add(rule.settle) + floor := since if !rule.witnessed { return floor } @@ -568,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri return judgeMoves(ctx, open, g, pairs, now) } +// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one +// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352). +func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool { + most := g.Since.Add(gateWithin) + return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most) +} + // judged is one module on one machine, as a gate judges it. type judged struct{ module, node string } @@ -578,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if g.Verdict != "" { return g.Verdict, nil } - if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery { + // A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing: + // a module healthy throughout is never held past it (novox/hq ADR 0352). + if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) { return "", nil } for _, c := range g.Carried { @@ -741,8 +754,8 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } g.Waits[m] = w } - // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the - // three readings' first two minutes after the send, the controller's one after the witness window. + // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the + // controller's and the tool runner's after the witness window, any other module's from the send. if now.Before(j.floors[m]) { continue } @@ -754,12 +767,13 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } } // What the send needs is what its most demanding module needs — the most readings, then the latest - // floor: a send carrying the node-engine is judged by its three readings. - wants, floor, said := 0, time.Time{}, "" + // floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout + // its two minutes (novox/hq ADR 0352). + wants, floor, within, said := 0, time.Time{}, time.Duration(0), "" for _, m := range modules { r, f := j.rules[m], j.floors[m] if r.passes > wants || (r.passes == wants && f.After(floor)) { - wants, said = r.passes, r.said + wants, within, said = r.passes, r.within, r.said } if f.After(floor) { floor = f @@ -768,14 +782,19 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { if len(modules) == 0 { // A send judged on no machine of its modules (its first machine not running them) has no class to // be judged by: it is judged as the core, never passed at once without a reading. - wants, floor = gatePasses, g.Since.Add(gateSettle) - said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle) + wants, floor, within = gatePasses, *g.Since, gateWithin + said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin) + } + // heldLongEnough is a rule's most: within after the send, healthy at every reading since it. + heldLongEnough := func(within time.Duration) bool { + return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within)) } g.Wants = wants // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes + r := j.rules[m] + return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within))) } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -792,6 +811,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { pastBound := now.Sub(*g.Since) > gateBound switch { case worst == healthBroken: + g.Faulted = true g.Read(now, false, g.BrokenWhy) var judging []string for _, m := range modules { @@ -810,9 +830,11 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — // the provider's own condition says what is wrong (ADR 0240 rule 5). + g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) case worst == healthNotYet: + g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) if pastBound { @@ -831,7 +853,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= wants { + if g.Passes >= wants || heldLongEnough(within) { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) } diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 6cac4ffc..b8377d77 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -21,15 +21,15 @@ import ( // wait, and a provider waiting for a login says who waits on it. // withGateBounds sets the gate's bounds for one test. -func withGateBounds(t *testing.T, settle, every, bound time.Duration) { +func withGateBounds(t *testing.T, within, every, bound time.Duration) { t.Helper() - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery, gateBound = settle, every, bound - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = within, every, bound + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) } // factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says @@ -60,7 +60,7 @@ func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() - withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) + withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second) since := time.Now().Add(-time.Second) base := factsOn(t, "laptop", since, healthyContainer("app")) wasGather := gatherGateFacts @@ -101,7 +101,7 @@ const broker_runtime = "node-tools" func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { open := aMesh(t) ctx := t.Context() - withGateBounds(t, 0, time.Hour, time.Hour) + withGateBounds(t, time.Hour, time.Hour, time.Hour) since := time.Now().Add(-time.Minute) base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) @@ -380,7 +380,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}} return f, nil } - withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second) + withGateBounds(t, time.Hour, 0, 10*time.Second) release := func() inventory.Plan { t.Helper() diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go index 3d23012f..79e7c491 100644 --- a/cmd/mesh-controller/gate_rule_test.go +++ b/cmd/mesh-controller/gate_rule_test.go @@ -10,19 +10,19 @@ import ( "github.com/novox/mesh-controller/internal/lease" ) -// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one -// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver -// keep three, the first two minutes after the send; the bus is kept as it was. +// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass +// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's +// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so. // withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. func withTheGatesTimes(t *testing.T) { t.Helper() - wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow + wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles - gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second + gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false t.Cleanup(func() { - gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow + gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles }) } @@ -33,7 +33,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) { module string claims []catalogue.Claim passes int - settle time.Duration + within time.Duration witnessed bool said string }{ @@ -46,8 +46,8 @@ func TestEachModuleHasItsGateRule(t *testing.T) { {"app", nil, 1, 0, false, "a leaf module's rule"}, } { r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { - t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle, + if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { + t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within, c.witnessed, c.said) } } @@ -161,38 +161,107 @@ func TestAControllerThatFailsIsStillFailed(t *testing.T) { } } -// The node-engine and the resolver keep three readings, the first at least two minutes after the send. -func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) { +// threeReadings judges a module under the three-reading rule on one machine, the send at sent. +func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) { + t.Helper() + g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent} + rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) + floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) + return g, func(after time.Duration, h health) { + why := "" + if h != healthGood { + why = "the node-engine reports another build" + } + tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why, + rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after)) + } +} + +// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two +// minutes (novox/hq ADR 0352). +func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) { withTheGatesTimes(t) for _, module := range []string{"mesh-host", "dnsmasq"} { - sent := time.Now() - o := aGateOn(module, "ace", sent) - rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) - floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) - judge := func(after time.Duration) { - tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood}, - worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, - sent.Add(after)) + g, judge := threeReadings(t, module, time.Now()) + judge(15*time.Second, healthGood) + judge(55*time.Second, healthGood) + if g.Verdict != "" || g.Passes != 2 { + t.Fatalf("%s: after two readings %+v; want still judging", module, g) } - for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} { - judge(after) - } - if o.g.Passes != 0 || o.g.Verdict != "" { - t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g) - } - judge(2 * time.Minute) - judge(160 * time.Second) - if o.g.Verdict != "" || o.g.Passes != 2 { - t.Fatalf("%s: after two readings %+v; want still judging", module, o.g) - } - judge(200 * time.Second) - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 || - !strings.Contains(o.g.Why, "the core's rule: 3 readings") { - t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g) + judge(95*time.Second, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 || + !strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") { + t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g) } } } +// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at +// two minutes comes sooner than the spacing after the last pass. +func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + last := sent.Add(100 * time.Second) + held := &inventory.PlanGate{Since: &sent, LastPass: &last} + if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) { + t.Fatal("the judging at two minutes is not due, or is due before") + } + held.Faulted = true + if dueAtTheMost(held, sent.Add(2*time.Minute)) { + t.Fatal("a send that was not healthy at a reading is judged at two minutes") + } + + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(100*time.Second, healthGood) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 2 { + t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) + } +} + +// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. +func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(10*time.Second, healthNotYet) + judge(100*time.Second, healthGood) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Faulted { + t.Fatalf("%+v; want passed at two minutes", g) + } +} + +// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one +// never healthy fails at the bound and is put back; one found broken fails. +func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(20*time.Second, healthNotYet) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthGood) + judge(2*time.Minute+10*time.Second, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 3 { + t.Fatalf("%+v; want passed only on its third healthy reading", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthNotYet) + judge(2*time.Minute, healthGood) + if g.Verdict != "" || !g.Faulted { + t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g) + } + judge(10*time.Minute+time.Second, healthNotYet) + if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") { + t.Fatalf("never healthy again: %+v; want failed at the bound", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthBroken) + if g.Verdict != inventory.GateFailed { + t.Fatalf("a broken reading: %+v; want failed, to be put back", g) + } +} + // The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { withTheGatesTimes(t) @@ -226,11 +295,11 @@ func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) } - for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} { + for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} { tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) } - if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 { - t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g) + if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 { + t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g) } } diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 06294a82..d8062644 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -124,14 +124,14 @@ func aGateMesh(t *testing.T) *gateMesh { } t.Cleanup(func() { gatherGateFacts = wasGather }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateSettle, gateEvery = 0, time.Hour - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", @@ -276,7 +276,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { g := aGateMesh(t) ctx := t.Context() - gateSettle = time.Hour // a core module could not pass for an hour + gateWithin = time.Hour // a core module could not pass on fewer readings for an hour advancePlans(ctx, g.open) // the first machine is sent the new build advancePlans(ctx, g.open) // one healthy judging p := g.plan(t) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 263a9f01..35b7d35c 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -133,15 +133,15 @@ func aBacklog(t *testing.T) *backlogMesh { } return f, nil } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery = 0, time.Hour + gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return b } diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 4fa3b172..53fb53ca 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -112,14 +112,14 @@ func TestReplay318(t *testing.T) { } wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) @@ -188,7 +188,7 @@ func TestReplay318(t *testing.T) { return f, nil } // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. - gateSettle, gateEvery, gateBound = 0, 0, 0 + gateWithin, gateEvery, gateBound = time.Hour, 0, 0 for i := 0; i < 12; i++ { advancePlans(ctx, open) diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 5a107b23..2798542d 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -248,13 +248,13 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { return names, nil } t.Cleanup(func() { sendRollout = wasSend }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery = 0, 0 - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, 0 + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index bf3ad9a4..45df338a 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -271,16 +271,16 @@ func aSettledMesh(t *testing.T) *walkMesh { } return f, nil } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateSettle, gateEvery = 0, time.Hour + gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather settingsDataMoves = wasData - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return w } diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index df003116..c8a21dcd 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -130,14 +130,14 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh { } t.Cleanup(func() { gatherGateFacts = wasGather }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateSettle, gateEvery = 0, time.Hour - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() states := map[string]*inventory.PlanModule{} @@ -409,7 +409,7 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } return f, err } - gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second + gateEvery, gateWithin, gateBound = 0, time.Hour, 10*time.Second for i := 0; i < 20 && len(seen) < 2; i++ { advancePlans(ctx, tm.open) } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index abc02bf6..a91a118f 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -330,11 +330,15 @@ type PlanGate struct { // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any - // other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate + // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two + // minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR + // 0282 decision 4 and ADR 0304 decision 3). Zero in a gate // judged before it was said: three. Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` + // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading + // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). + Faulted bool `json:"faulted,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"` // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since. From 41eefc2f8c3eb4f152aefb7ebb9fc9f537d4d3b3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 22:58:37 +0200 Subject: [PATCH 3/4] Test that a healthy node-engine passes at two minutes on one reading --- cmd/mesh-controller/gate_rule_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go index 79e7c491..2396937f 100644 --- a/cmd/mesh-controller/gate_rule_test.go +++ b/cmd/mesh-controller/gate_rule_test.go @@ -217,6 +217,11 @@ func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { if g.Verdict != inventory.GatePassed || g.Passes != 2 { t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 1 { + t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g) + } } // A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. From 675466a8c35f0c700098161ae7d1f3856e533269 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 23:07:15 +0200 Subject: [PATCH 4/4] Hold the store tests' modules as the settle time did, and fault a dip after any healthy reading Two tier tests relied on the two-minute settle to keep a module unjudged; the witness window and the gate's spacing now hold it. A healthy reading taken before the controller's floor counted no pass, so a dip after it did not take the two-minute pass away (novox/hq ADR 0352 decision 3). --- cmd/mesh-controller/gate.go | 6 ++++-- cmd/mesh-controller/gate_rule_test.go | 28 +++++++++++++++++++++++++++ cmd/mesh-controller/tier_send_test.go | 8 +++++++- internal/inventory/plans.go | 3 +++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index de910f6e..05a2d4d0 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -830,11 +830,11 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — // the provider's own condition says what is wrong (ADR 0240 rule 5). - g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting + g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) case worst == healthNotYet: - g.Faulted = g.Faulted || g.Passes > 0 // not healthy after a healthy reading; before one, still starting + g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) if pastBound { @@ -842,6 +842,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } case now.Before(floor): // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). + g.HealthySeen = true // The bound still holds: a module that cannot be read healthy after its floor fails at it. g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", floor.UTC().Format(time.RFC3339), said), nil @@ -850,6 +851,7 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. + g.HealthySeen = true g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go index 2396937f..06777685 100644 --- a/cmd/mesh-controller/gate_rule_test.go +++ b/cmd/mesh-controller/gate_rule_test.go @@ -289,6 +289,34 @@ func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { } } +// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at +// two minutes (novox/hq ADR 0352 decision 3). +func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + } + floors["mesh-host"] = sent + floors["mesh-controller"] = sent.Add(90 * time.Second) + judge := func(after time.Duration, h health) { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h}, + worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after)) + } + judge(30*time.Second, healthGood) + if g.Passes != 0 { + t.Fatalf("a reading before the controller's floor was counted: %+v", g) + } + judge(70*time.Second, healthNotYet) + judge(2*time.Minute, healthGood) + if !g.Faulted || g.Verdict != "" { + t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g) + } +} + // A send carrying the controller and the node-engine is judged by the node-engine's three readings. func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { withTheGatesTimes(t) diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index c8a21dcd..97241324 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -226,6 +226,9 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) ctx := t.Context() inv := tm.open.inventory + // The node tools are held inside their witness window, which the node-engine keeps while they settle: their + // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). + gateWitnessWindow = time.Hour advancePlans(ctx, tm.open) if len(tm.sent) != 1 { @@ -409,7 +412,10 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } return f, err } - gateEvery, gateWithin, gateBound = 0, time.Hour, 10*time.Second + // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the + // judgings that find the node tools broken, and is judged to its own pass after them (the hold the + // settle time gave before novox/hq ADR 0352 counted readings from the send). + gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second for i := 0; i < 20 && len(seen) < 2; i++ { advancePlans(ctx, tm.open) } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index a91a118f..5f058270 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -339,6 +339,9 @@ type PlanGate struct { // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). Faulted bool `json:"faulted,omitempty"` + // HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor + // counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3). + HealthySeen bool `json:"healthy_seen,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"` // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.