From 524295926f76bee5516ac664993d8038e821b3b9 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 22:39:32 +0200 Subject: [PATCH] Pass the controller's first-node gate on one reading after the witness window The controller runs on one machine, so three readings over two minutes guarded no other machine and held every walk behind it. Judge each module by its own rule: one reading for the controller and the tool runner once the node-engine's witness window has ended, one for mesh-delivery, three for the node-engine and the resolver with the first two minutes after the send (novox/hq ADR 0351). The gate's verdict names the rule it applied. --- cmd/mesh-controller/found_wait_test.go | 4 + cmd/mesh-controller/gate.go | 196 ++++++++++++++---- cmd/mesh-controller/gate_followup_test.go | 4 + cmd/mesh-controller/gate_rule_test.go | 236 ++++++++++++++++++++++ cmd/mesh-controller/gate_test.go | 29 +-- cmd/mesh-controller/release_test.go | 4 + cmd/mesh-controller/replay318_test.go | 4 + cmd/mesh-controller/replays_test.go | 4 + cmd/mesh-controller/settings_walk_test.go | 4 + cmd/mesh-controller/tier_send_test.go | 25 ++- internal/inventory/plans.go | 9 +- 11 files changed, 448 insertions(+), 71 deletions(-) create mode 100644 cmd/mesh-controller/gate_rule_test.go diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index 3fff9be3..81c27e70 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -82,6 +82,10 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { return f, err } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index f5aa1689..c140bd78 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -28,9 +28,11 @@ import ( // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the -// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then -// are the rest sent. +// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the +// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the +// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states +// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3). +// Only then are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // at its gate — registration refuses it and nothing sends it again on its own — the module's registered @@ -41,22 +43,28 @@ import ( // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. var ( - // gateSettle is how long the first machine must stay healthy, at the least. + // gateSettle is how long after the send a core module judged by three readings is first read, at the least. gateSettle = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the - // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every - // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304 - // decision 3 changed it, issue 524). + // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at + // the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the + // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 + // decision 4 as ADR 0304 decision 3 changed it, issue 524). gatePasses = 3 gateLeafPasses = 1 gateLeafSettles = false gateEvery = 40 * time.Second + // gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its + // start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either + // counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR + // 0351 decisions 1 and 4). + gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin) ) // coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's -// own path, or one holding the mesh's resolver. Every other module is a leaf. +// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's +// delivery budget; how many readings its gate takes is gateRuleOf's. func coreModule(module string, m catalogue.Manifest) bool { if _, own := onTheControllersPath[module]; own { return true @@ -69,12 +77,71 @@ func coreModule(module string, m catalogue.Manifest) bool { return coreComponent(module) != "" } -// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first. -func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) { - if coreModule(module, m) { - return gatePasses, true +// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the +// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's +// witness window for it. said names the rule in the gate's text. +type gateRule struct { + passes int + settle time.Duration + witnessed bool + said string +} + +// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its +// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send, +// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never +// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner, +// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4); +// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304 +// decision 3). +func gateRuleOf(module string, m catalogue.Manifest) gateRule { + switch { + case coreComponent(module) == lease.ComponentController: + return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case coreComponent(module) == lease.ComponentNodeTools: + return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case module == catalogue.DeliverySeat: + return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} + case coreModule(module, m): + return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+ + "the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)} } - return gateLeafPasses, gateLeafSettles + r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} + if gateLeafSettles { + r.settle = gateSettle + r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send", + gateLeafPasses, gateSettle) + } + return r +} + +// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed +// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 +// and 4). The window opens when the node-engine starts the build, which is before the machine reports it +// applied (applied, as the gate first read it) and, for the controller, before the new controller took the +// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. +// A controller that starts again takes the lease again, and so moves its floor on. Pure. +func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, + holder *lease.Holder) time.Time { + floor := since.Add(rule.settle) + if !rule.witnessed { + return floor + } + for _, n := range machines { + start := since + if at, read := applied[n]; read && at.After(start) { + start = at + } + if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) { + start = holder.Taken + } + if end := start.Add(gateWitnessWindow); end.After(floor) { + floor = end + } + } + return floor } // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions @@ -608,6 +675,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs why = said } } + // **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read + // each machine report its send applied, and who holds the controller lease, the earliest its reading counts. + if g.Applied == nil { + g.Applied = map[string]time.Time{} + } + for node := range byMachine { + r, said := facts.reports[node] + if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied && + facts.reportedOn(node, r) { + g.Applied[node] = *r.At + } + } + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + var on []string + for _, j := range pairs { + if j.module == m { + on = append(on, j.node) + } + } + rules[m] = gateRuleOf(m, shelf[m]) + floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder) + } + tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why, + rules: rules, floors: floors}, now) + return g.Verdict, nil +} + +// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading +// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's +// rule and the earliest its reading counts. +type gateJudging struct { + modules []string + reading map[string]health + waits map[string]string + worst health + why string + rules map[string]gateRule + floors map[string]time.Time +} + +// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but +// for the gate it writes. +func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { + modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why if g.Healthy == nil { g.Healthy = map[string]int{} } @@ -623,38 +735,47 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs delete(g.HealthyAt, m) continue } - // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more - // often while another module beside it is not yet healthy counts no faster. - if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { - g.Healthy[m]++ - g.HealthyAt[m] = now - } if w := waits[m]; w != "" { if g.Waits == nil { g.Waits = map[string]string{} } g.Waits[m] = w } + // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the + // three readings' first two minutes after the send, the controller's one after the witness window. + if now.Before(j.floors[m]) { + continue + } + // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more + // often while another module beside it is not yet healthy counts no faster. + if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { + g.Healthy[m]++ + g.HealthyAt[m] = now + } } - settled := now.Sub(*g.Since) >= gateSettle - // What the send needs is what its most demanding module needs: a send carrying a core module is judged - // as the core. - wants, settles := 0, false + // What the send needs is what its most demanding module needs — the most readings, then the latest + // floor: a send carrying the node-engine is judged by its three readings. + wants, floor, said := 0, time.Time{}, "" for _, m := range modules { - p, s := gateWants(m, shelf[m]) - wants, settles = max(wants, p), settles || s + r, f := j.rules[m], j.floors[m] + if r.passes > wants || (r.passes == wants && f.After(floor)) { + wants, said = r.passes, r.said + } + if f.After(floor) { + floor = f + } } if len(modules) == 0 { // A send judged on no machine of its modules (its first machine not running them) has no class to // be judged by: it is judged as the core, never passed at once without a reading. - wants, settles = gatePasses, true + wants, floor = gatePasses, g.Since.Add(gateSettle) + said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle) } g.Wants = wants - // A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after + // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - p, s := gateWants(m, shelf[m]) - return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p + return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -697,17 +818,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if pastBound { fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) } + case now.Before(floor): + // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). + // The bound still holds: a module that cannot be read healthy after its floor fails at it. + g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", + floor.UTC().Format(time.RFC3339), said), nil + if pastBound { + fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last)) + } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= wants && (settled || !settles) { - decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, - now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) + if g.Passes >= wants { + decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, + now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) } } - return g.Verdict, nil } // judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 35929eb5..6cac4ffc 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -24,6 +24,10 @@ import ( func withGateBounds(t *testing.T, settle, every, bound time.Duration) { t.Helper() wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery, gateBound = settle, every, bound t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) } diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go new file mode 100644 index 00000000..3d23012f --- /dev/null +++ b/cmd/mesh-controller/gate_rule_test.go @@ -0,0 +1,236 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one +// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver +// keep three, the first two minutes after the send; the bus is kept as it was. + +// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. +func withTheGatesTimes(t *testing.T) { + t.Helper() + wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow + wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles + gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second + gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false + t.Cleanup(func() { + gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow + gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles + }) +} + +func TestEachModuleHasItsGateRule(t *testing.T) { + withTheGatesTimes(t) + for _, c := range []struct { + module string + claims []catalogue.Claim + passes int + settle time.Duration + witnessed bool + said string + }{ + {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, + {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, + {catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"}, + {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, + {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"app", nil, 1, 0, false, "a leaf module's rule"}, + } { + r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) + if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { + t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle, + c.witnessed, c.said) + } + } +} + +// Every module that was core stays core for its delivery budget and its walk: only its readings change. +func TestTheCoreStaysTheCore(t *testing.T) { + for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { + if !coreModule(m, catalogue.Manifest{Module: m}) { + t.Errorf("%s is no longer judged as the core for its class", m) + } + } + if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { + t.Error("the resolver's holder is no longer judged as the core for its class") + } +} + +// gateOn judges one module on one machine, healthy at every reading. +type gateOn struct { + module, node string + g *inventory.PlanGate + holder *lease.Holder +} + +func aGateOn(module, node string, since time.Time) *gateOn { + return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, + Applied: map[string]time.Time{}}} +} + +func (o *gateOn) read(at time.Time, h health, why string) { + r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module}) + floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) + tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, + worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) +} + +func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(15 * time.Second) + o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} + + // Ready, but inside the witness window, which opened before it took the lease: not counted. + for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { + o.read(sent.Add(after), healthGood, "") + if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { + t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) + } + } + o.read(sent.Add(80*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { + t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) + } + if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || + !strings.Contains(o.g.Why, "ADR 0351") { + t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) + } +} + +// A controller that starts again takes the lease again: its witness window starts again, and so does the +// gate's wait. +func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(10 * time.Second) + o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} + o.read(sent.Add(90*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) + } + o.read(sent.Add(130*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed { + t.Fatalf("not passed after the new window: %+v", o.g) + } +} + +// Without the lease's word (a controller carried by another gate), the window counts from the applied report, +// and from the send at the least. +func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + r := gateRuleOf("mesh-controller", catalogue.Manifest{}) + if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { + t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) + } + if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, + map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { + t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) + } +} + +// A controller that a witness put back, or that never becomes ready, fails its gate as before. +func TestAControllerThatFailsIsStillFailed(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") + if o.g.Verdict != inventory.GateFailed { + t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) + } + o = aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != "" { + t.Fatalf("failed before the bound: %+v", o.g) + } + o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { + t.Fatalf("never ready: %+v; want failed at the bound", o.g) + } +} + +// The node-engine and the resolver keep three readings, the first at least two minutes after the send. +func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) { + withTheGatesTimes(t) + for _, module := range []string{"mesh-host", "dnsmasq"} { + sent := time.Now() + o := aGateOn(module, "ace", sent) + rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) + floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) + judge := func(after time.Duration) { + tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood}, + worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, + sent.Add(after)) + } + for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} { + judge(after) + } + if o.g.Passes != 0 || o.g.Verdict != "" { + t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g) + } + judge(2 * time.Minute) + judge(160 * time.Second) + if o.g.Verdict != "" || o.g.Passes != 2 { + t.Fatalf("%s: after two readings %+v; want still judging", module, o.g) + } + judge(200 * time.Second) + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 || + !strings.Contains(o.g.Why, "the core's rule: 3 readings") { + t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g) + } + } +} + +// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. +func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("node-tools", "ace", sent) + o.g.Applied["ace"] = sent.Add(10 * time.Second) + o.read(sent.Add(30*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) + } + o.read(sent.Add(70*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { + t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) + } + + o = aGateOn(catalogue.DeliverySeat, "novox", sent) + o.read(sent.Add(5*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { + t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) + } +} + +// A send carrying the controller and the node-engine is judged by the node-engine's three readings. +func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) + } + for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, + "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) + } + if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 { + t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g) + } +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index ced2ccce..06294a82 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -125,6 +125,10 @@ func aGateMesh(t *testing.T) *gateMesh { t.Cleanup(func() { gatherGateFacts = wasGather }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) @@ -285,31 +289,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { } } -// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three -// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3). -func TestAModuleIsJudgedByItsClass(t *testing.T) { - for _, c := range []struct { - module string - claims []catalogue.Claim - passes int - settles bool - }{ - {"app", nil, 1, false}, - {"nzbget", nil, 1, false}, - {"mesh-controller", nil, 3, true}, - {"mesh-host", nil, 3, true}, - {"node-tools", nil, 3, true}, - {"nats", nil, 3, true}, - {catalogue.DeliverySeat, nil, 3, true}, - {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true}, - } { - p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if p != c.passes || s != c.settles { - t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles) - } - } -} - // A first machine that never becomes healthy fails its gate at the bound, and is put back. func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { judgeLeavesAsTheCore(t) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 0560cdd8..263a9f01 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -134,6 +134,10 @@ func aBacklog(t *testing.T) *backlogMesh { return f, nil } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 09e212d0..4fa3b172 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -113,6 +113,10 @@ func TestReplay318(t *testing.T) { wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 138f0180..5a107b23 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -249,6 +249,10 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { } t.Cleanup(func() { sendRollout = wasSend }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, 0 t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 48c34f2e..bf3ad9a4 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -272,6 +272,10 @@ func aSettledMesh(t *testing.T) *walkMesh { return f, nil } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index c91a8fc0..df003116 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -131,6 +131,10 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh { t.Cleanup(func() { gatherGateFacts = wasGather }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) @@ -440,15 +444,15 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } } -// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node -// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three -// readings. -func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { +// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 +// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool +// runner's rule: one reading, not before the node-engine's witness window for it has ended. +func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { tm := aTierMesh(t, broker.RuntimeModule, "app1") ctx := t.Context() // One reading counted per advance, as every test with a gate left open does: a gate judged with no time // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateSettle = time.Hour, time.Hour + gateEvery, gateWitnessWindow = time.Hour, time.Hour for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -457,10 +461,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { if lead == nil { lead = p.Modules["app1"].Gate } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses { - t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { + t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) } - gateEvery, gateSettle = 0, 0 + gateEvery, gateWitnessWindow = 0, 0 for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -469,8 +473,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) } for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) { - t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses) + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || + !strings.Contains(g.Why, "the tool runner's rule")) { + t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) } } } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index c17344a0..abc02bf6 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -323,11 +323,16 @@ type PlanGate struct { // machine's report is held against. Absent on a gate kept before it was, which reads the report // against the send made last, as before. Sent map[string]SentDeclaration `json:"sent,omitempty"` + // Applied is, per machine, the time of the first report the gate read of its send applied: the + // node-engine started the build before it, so a witness window counted from it ends no earlier than the + // node-engine's own (novox/hq ADR 0351 decisions 1 and 4). + Applied map[string]time.Time `json:"applied,omitempty"` // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282 - // decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three. + // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any + // other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate + // judged before it was said: three. Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` // Last is what the newest judging found wanting, while it still may pass.