Pass the controller's first-node gate on one reading after the witness window
mesh/merge-gate head 524295926f76 merged into trunk bff5b7f0e4f8 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 524295926f76 merged into trunk bff5b7f0e4f8 — fail: its merge-check.sh failed: --- FAIL: TestACoreBehindWhileTheNodeToolsSettleFailsNo…
mesh/delivery superseded: a newer head of the same pull request

The controller runs on one machine, so three readings over two minutes guarded no
other machine and held every walk behind it. Judge each module by its own rule:
one reading for the controller and the tool runner once the node-engine's witness
window has ended, one for mesh-delivery, three for the node-engine and the
resolver with the first two minutes after the send (novox/hq ADR 0351). The
gate's verdict names the rule it applied.
This commit is contained in:
2026-10-13 22:39:32 +02:00
parent bff5b7f0e4
commit 524295926f
11 changed files with 448 additions and 71 deletions
+4
View File
@@ -82,6 +82,10 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
return f, err
}
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
deadline := time.Now().Add(5 * time.Second)
+162 -34
View File
@@ -28,9 +28,11 @@ import (
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
// are the rest sent.
// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the
// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the
// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3).
// Only then are the rest sent.
//
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -41,22 +43,28 @@ import (
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
var (
// gateSettle is how long the first machine must stay healthy, at the least.
// gateSettle is how long after the send a core module judged by three readings is first read, at the least.
gateSettle = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
// decision 3 changed it, issue 524).
// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
// the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the
// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
// decision 4 as ADR 0304 decision 3 changed it, issue 524).
gatePasses = 3
gateLeafPasses = 1
gateLeafSettles = false
gateEvery = 40 * time.Second
// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
// 0351 decisions 1 and 4).
gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
)
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
// own path, or one holding the mesh's resolver. Every other module is a leaf.
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
// delivery budget; how many readings its gate takes is gateRuleOf's.
func coreModule(module string, m catalogue.Manifest) bool {
if _, own := onTheControllersPath[module]; own {
return true
@@ -69,12 +77,71 @@ func coreModule(module string, m catalogue.Manifest) bool {
return coreComponent(module) != ""
}
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
if coreModule(module, m) {
return gatePasses, true
// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's
// witness window for it. said names the rule in the gate's text.
type gateRule struct {
passes int
settle time.Duration
witnessed bool
said string
}
// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
// decision 3).
func gateRuleOf(module string, m catalogue.Manifest) gateRule {
switch {
case coreComponent(module) == lease.ComponentController:
return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
"after the node-engine's witness window (novox/hq ADR 0351)"}
case coreComponent(module) == lease.ComponentNodeTools:
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
"after the node-engine's witness window (novox/hq ADR 0351)"}
case module == catalogue.DeliverySeat:
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
case coreModule(module, m):
return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+
"the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)}
}
return gateLeafPasses, gateLeafSettles
r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
if gateLeafSettles {
r.settle = gateSettle
r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send",
gateLeafPasses, gateSettle)
}
return r
}
// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed
// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1
// and 4). The window opens when the node-engine starts the build, which is before the machine reports it
// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
// A controller that starts again takes the lease again, and so moves its floor on. Pure.
func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
holder *lease.Holder) time.Time {
floor := since.Add(rule.settle)
if !rule.witnessed {
return floor
}
for _, n := range machines {
start := since
if at, read := applied[n]; read && at.After(start) {
start = at
}
if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
start = holder.Taken
}
if end := start.Add(gateWitnessWindow); end.After(floor) {
floor = end
}
}
return floor
}
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
@@ -608,6 +675,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
why = said
}
}
// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
if g.Applied == nil {
g.Applied = map[string]time.Time{}
}
for node := range byMachine {
r, said := facts.reports[node]
if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
facts.reportedOn(node, r) {
g.Applied[node] = *r.At
}
}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
var on []string
for _, j := range pairs {
if j.module == m {
on = append(on, j.node)
}
}
rules[m] = gateRuleOf(m, shelf[m])
floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
}
tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
rules: rules, floors: floors}, now)
return g.Verdict, nil
}
// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
// rule and the earliest its reading counts.
type gateJudging struct {
modules []string
reading map[string]health
waits map[string]string
worst health
why string
rules map[string]gateRule
floors map[string]time.Time
}
// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
// for the gate it writes.
func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
if g.Healthy == nil {
g.Healthy = map[string]int{}
}
@@ -623,38 +735,47 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
delete(g.HealthyAt, m)
continue
}
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
// often while another module beside it is not yet healthy counts no faster.
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
g.Healthy[m]++
g.HealthyAt[m] = now
}
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
}
// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the
// three readings' first two minutes after the send, the controller's one after the witness window.
if now.Before(j.floors[m]) {
continue
}
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
// often while another module beside it is not yet healthy counts no faster.
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
g.Healthy[m]++
g.HealthyAt[m] = now
}
}
settled := now.Sub(*g.Since) >= gateSettle
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
// as the core.
wants, settles := 0, false
// What the send needs is what its most demanding module needs — the most readings, then the latest
// floor: a send carrying the node-engine is judged by its three readings.
wants, floor, said := 0, time.Time{}, ""
for _, m := range modules {
p, s := gateWants(m, shelf[m])
wants, settles = max(wants, p), settles || s
r, f := j.rules[m], j.floors[m]
if r.passes > wants || (r.passes == wants && f.After(floor)) {
wants, said = r.passes, r.said
}
if f.After(floor) {
floor = f
}
}
if len(modules) == 0 {
// A send judged on no machine of its modules (its first machine not running them) has no class to
// be judged by: it is judged as the core, never passed at once without a reading.
wants, settles = gatePasses, true
wants, floor = gatePasses, g.Since.Add(gateSettle)
said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle)
}
g.Wants = wants
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
passedAlone := func(m string) bool {
p, s := gateWants(m, shelf[m])
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes
}
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
fail := func(why string) {
@@ -697,17 +818,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
if pastBound {
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
}
case now.Before(floor):
// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
// The bound still holds: a module that cannot be read healthy after its floor fails at it.
g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
floor.UTC().Format(time.RFC3339), said), nil
if pastBound {
fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
}
default:
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
g.Read(now, true, "")
if g.Passes >= wants && (settled || !settles) {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
if g.Passes >= wants {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
}
}
return g.Verdict, nil
}
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
@@ -24,6 +24,10 @@ import (
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
t.Helper()
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateSettle, gateEvery, gateBound = settle, every, bound
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
}
+236
View File
@@ -0,0 +1,236 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one
// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver
// keep three, the first two minutes after the send; the bus is kept as it was.
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
func withTheGatesTimes(t *testing.T) {
t.Helper()
wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
t.Cleanup(func() {
gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
})
}
func TestEachModuleHasItsGateRule(t *testing.T) {
withTheGatesTimes(t)
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
settle time.Duration
witnessed bool
said string
}{
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"app", nil, 1, 0, false, "a leaf module's rule"},
} {
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle,
c.witnessed, c.said)
}
}
}
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
func TestTheCoreStaysTheCore(t *testing.T) {
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
if !coreModule(m, catalogue.Manifest{Module: m}) {
t.Errorf("%s is no longer judged as the core for its class", m)
}
}
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
t.Error("the resolver's holder is no longer judged as the core for its class")
}
}
// gateOn judges one module on one machine, healthy at every reading.
type gateOn struct {
module, node string
g *inventory.PlanGate
holder *lease.Holder
}
func aGateOn(module, node string, since time.Time) *gateOn {
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
Applied: map[string]time.Time{}}}
}
func (o *gateOn) read(at time.Time, h health, why string) {
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
}
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(15 * time.Second)
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
// Ready, but inside the witness window, which opened before it took the lease: not counted.
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
o.read(sent.Add(after), healthGood, "")
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
}
}
o.read(sent.Add(80*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
}
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
!strings.Contains(o.g.Why, "ADR 0351") {
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
}
}
// A controller that starts again takes the lease again: its witness window starts again, and so does the
// gate's wait.
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(10 * time.Second)
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
o.read(sent.Add(90*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
}
o.read(sent.Add(130*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed {
t.Fatalf("not passed after the new window: %+v", o.g)
}
}
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
// and from the send at the least.
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
}
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
}
}
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
if o.g.Verdict != inventory.GateFailed {
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
}
o = aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != "" {
t.Fatalf("failed before the bound: %+v", o.g)
}
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
}
}
// The node-engine and the resolver keep three readings, the first at least two minutes after the send.
func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) {
withTheGatesTimes(t)
for _, module := range []string{"mesh-host", "dnsmasq"} {
sent := time.Now()
o := aGateOn(module, "ace", sent)
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
judge := func(after time.Duration) {
tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood},
worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}},
sent.Add(after))
}
for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} {
judge(after)
}
if o.g.Passes != 0 || o.g.Verdict != "" {
t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g)
}
judge(2 * time.Minute)
judge(160 * time.Second)
if o.g.Verdict != "" || o.g.Passes != 2 {
t.Fatalf("%s: after two readings %+v; want still judging", module, o.g)
}
judge(200 * time.Second)
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 ||
!strings.Contains(o.g.Why, "the core's rule: 3 readings") {
t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g)
}
}
}
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("node-tools", "ace", sent)
o.g.Applied["ace"] = sent.Add(10 * time.Second)
o.read(sent.Add(30*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
}
o.read(sent.Add(70*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
}
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
o.read(sent.Add(5*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
}
}
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
modules := []string{"mesh-controller", "mesh-host"}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
}
for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} {
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
}
if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 {
t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g)
}
}
+4 -25
View File
@@ -125,6 +125,10 @@ func aGateMesh(t *testing.T) *gateMesh {
t.Cleanup(func() { gatherGateFacts = wasGather })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
@@ -285,31 +289,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
}
}
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
func TestAModuleIsJudgedByItsClass(t *testing.T) {
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
settles bool
}{
{"app", nil, 1, false},
{"nzbget", nil, 1, false},
{"mesh-controller", nil, 3, true},
{"mesh-host", nil, 3, true},
{"node-tools", nil, 3, true},
{"nats", nil, 3, true},
{catalogue.DeliverySeat, nil, 3, true},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
} {
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if p != c.passes || s != c.settles {
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
}
}
}
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
judgeLeavesAsTheCore(t)
+4
View File
@@ -134,6 +134,10 @@ func aBacklog(t *testing.T) *backlogMesh {
return f, nil
}
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
+4
View File
@@ -113,6 +113,10 @@ func TestReplay318(t *testing.T) {
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
+4
View File
@@ -249,6 +249,10 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
}
t.Cleanup(func() { sendRollout = wasSend })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateSettle, gateEvery = 0, 0
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
@@ -272,6 +272,10 @@ func aSettledMesh(t *testing.T) *walkMesh {
return f, nil
}
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
+15 -10
View File
@@ -131,6 +131,10 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh {
t.Cleanup(func() { gatherGateFacts = wasGather })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
@@ -440,15 +444,15 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
}
}
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
// readings.
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351
// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool
// runner's rule: one reading, not before the node-engine's witness window for it has ended.
func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
tm := aTierMesh(t, broker.RuntimeModule, "app1")
ctx := t.Context()
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
// between readings would count a pass at every look and never let an advance rest.
gateEvery, gateSettle = time.Hour, time.Hour
gateEvery, gateWitnessWindow = time.Hour, time.Hour
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
@@ -457,10 +461,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
if lead == nil {
lead = p.Modules["app1"].Gate
}
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 {
t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
}
gateEvery, gateSettle = 0, 0
gateEvery, gateWitnessWindow = 0, 0
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
@@ -469,8 +473,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
}
for _, m := range []string{broker.RuntimeModule, "app1"} {
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 ||
!strings.Contains(g.Why, "the tool runner's rule")) {
t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g)
}
}
}
+7 -2
View File
@@ -323,11 +323,16 @@ type PlanGate struct {
// machine's report is held against. Absent on a gate kept before it was, which reads the report
// against the send made last, as before.
Sent map[string]SentDeclaration `json:"sent,omitempty"`
// Applied is, per machine, the time of the first report the gate read of its send applied: the
// node-engine started the build before it, so a witness window counted from it ends no earlier than the
// node-engine's own (novox/hq ADR 0351 decisions 1 and 4).
Applied map[string]time.Time `json:"applied,omitempty"`
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
// does not resets them.
Passes int `json:"passes,omitempty"`
// Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282
// decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three.
// Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any
// other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate
// judged before it was said: three.
Wants int `json:"wants,omitempty"`
LastPass *time.Time `json:"last_pass,omitempty"`
// Last is what the newest judging found wanting, while it still may pass.