diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index 3fff9be3..81c27e70 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -82,6 +82,10 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { return f, err } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index f5aa1689..c140bd78 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -28,9 +28,11 @@ import ( // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the -// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then -// are the rest sent. +// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing three times, the +// first two minutes after the send; the controller and the tool runner once, no earlier than the end of the +// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states +// it healthy (novox/hq ADR 0351, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 decision 3). +// Only then are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // at its gate — registration refuses it and nothing sends it again on its own — the module's registered @@ -41,22 +43,28 @@ import ( // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. var ( - // gateSettle is how long the first machine must stay healthy, at the least. + // gateSettle is how long after the send a core module judged by three readings is first read, at the least. gateSettle = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the - // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every - // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304 - // decision 3 changed it, issue 524). + // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at + // the least, the first gateSettle after the send. gateLeafPasses is a leaf module's: one, taken once the + // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 + // decision 4 as ADR 0304 decision 3 changed it, issue 524). gatePasses = 3 gateLeafPasses = 1 gateLeafSettles = false gateEvery = 40 * time.Second + // gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its + // start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either + // counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR + // 0351 decisions 1 and 4). + gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin) ) // coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's -// own path, or one holding the mesh's resolver. Every other module is a leaf. +// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's +// delivery budget; how many readings its gate takes is gateRuleOf's. func coreModule(module string, m catalogue.Manifest) bool { if _, own := onTheControllersPath[module]; own { return true @@ -69,12 +77,71 @@ func coreModule(module string, m catalogue.Manifest) bool { return coreComponent(module) != "" } -// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first. -func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) { - if coreModule(module, m) { - return gatePasses, true +// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the +// least, the first no earlier than settle after the send and, where witnessed, the end of the node-engine's +// witness window for it. said names the rule in the gate's text. +type gateRule struct { + passes int + settle time.Duration + witnessed bool + said string +} + +// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its +// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send, +// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never +// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner, +// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4); +// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304 +// decision 3). +func gateRuleOf(module string, m catalogue.Manifest) gateRule { + switch { + case coreComponent(module) == lease.ComponentController: + return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case coreComponent(module) == lease.ComponentNodeTools: + return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case module == catalogue.DeliverySeat: + return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} + case coreModule(module, m): + return gateRule{passes: gatePasses, settle: gateSettle, said: fmt.Sprintf("the core's rule: %d readings, "+ + "the first %s after the send (novox/hq ADR 0236, kept by ADR 0351)", gatePasses, gateSettle)} } - return gateLeafPasses, gateLeafSettles + r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} + if gateLeafSettles { + r.settle = gateSettle + r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, the first %s after the send", + gateLeafPasses, gateSettle) + } + return r +} + +// gateFloor is the earliest a module's reading counts by its rule: settle after the send and, for a witnessed +// module, the end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 +// and 4). The window opens when the node-engine starts the build, which is before the machine reports it +// applied (applied, as the gate first read it) and, for the controller, before the new controller took the +// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. +// A controller that starts again takes the lease again, and so moves its floor on. Pure. +func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, + holder *lease.Holder) time.Time { + floor := since.Add(rule.settle) + if !rule.witnessed { + return floor + } + for _, n := range machines { + start := since + if at, read := applied[n]; read && at.After(start) { + start = at + } + if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) { + start = holder.Taken + } + if end := start.Add(gateWitnessWindow); end.After(floor) { + floor = end + } + } + return floor } // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions @@ -608,6 +675,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs why = said } } + // **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read + // each machine report its send applied, and who holds the controller lease, the earliest its reading counts. + if g.Applied == nil { + g.Applied = map[string]time.Time{} + } + for node := range byMachine { + r, said := facts.reports[node] + if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied && + facts.reportedOn(node, r) { + g.Applied[node] = *r.At + } + } + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + var on []string + for _, j := range pairs { + if j.module == m { + on = append(on, j.node) + } + } + rules[m] = gateRuleOf(m, shelf[m]) + floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder) + } + tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why, + rules: rules, floors: floors}, now) + return g.Verdict, nil +} + +// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading +// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's +// rule and the earliest its reading counts. +type gateJudging struct { + modules []string + reading map[string]health + waits map[string]string + worst health + why string + rules map[string]gateRule + floors map[string]time.Time +} + +// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but +// for the gate it writes. +func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { + modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why if g.Healthy == nil { g.Healthy = map[string]int{} } @@ -623,38 +735,47 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs delete(g.HealthyAt, m) continue } - // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more - // often while another module beside it is not yet healthy counts no faster. - if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { - g.Healthy[m]++ - g.HealthyAt[m] = now - } if w := waits[m]; w != "" { if g.Waits == nil { g.Waits = map[string]string{} } g.Waits[m] = w } + // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 2): the + // three readings' first two minutes after the send, the controller's one after the witness window. + if now.Before(j.floors[m]) { + continue + } + // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more + // often while another module beside it is not yet healthy counts no faster. + if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { + g.Healthy[m]++ + g.HealthyAt[m] = now + } } - settled := now.Sub(*g.Since) >= gateSettle - // What the send needs is what its most demanding module needs: a send carrying a core module is judged - // as the core. - wants, settles := 0, false + // What the send needs is what its most demanding module needs — the most readings, then the latest + // floor: a send carrying the node-engine is judged by its three readings. + wants, floor, said := 0, time.Time{}, "" for _, m := range modules { - p, s := gateWants(m, shelf[m]) - wants, settles = max(wants, p), settles || s + r, f := j.rules[m], j.floors[m] + if r.passes > wants || (r.passes == wants && f.After(floor)) { + wants, said = r.passes, r.said + } + if f.After(floor) { + floor = f + } } if len(modules) == 0 { // A send judged on no machine of its modules (its first machine not running them) has no class to // be judged by: it is judged as the core, never passed at once without a reading. - wants, settles = gatePasses, true + wants, floor = gatePasses, g.Since.Add(gateSettle) + said = fmt.Sprintf("the core's rule: %d readings, the first %s after the send", gatePasses, gateSettle) } g.Wants = wants - // A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after + // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - p, s := gateWants(m, shelf[m]) - return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p + return reading[m] <= healthPerson && g.Healthy[m] >= j.rules[m].passes } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -697,17 +818,24 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if pastBound { fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) } + case now.Before(floor): + // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). + // The bound still holds: a module that cannot be read healthy after its floor fails at it. + g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", + floor.UTC().Format(time.RFC3339), said), nil + if pastBound { + fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last)) + } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= wants && (settled || !settles) { - decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, - now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) + if g.Passes >= wants { + decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, + now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) } } - return g.Verdict, nil } // judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 35929eb5..6cac4ffc 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -24,6 +24,10 @@ import ( func withGateBounds(t *testing.T, settle, every, bound time.Duration) { t.Helper() wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery, gateBound = settle, every, bound t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) } diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go new file mode 100644 index 00000000..3d23012f --- /dev/null +++ b/cmd/mesh-controller/gate_rule_test.go @@ -0,0 +1,236 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// The first-node gate's readings by module (novox/hq ADR 0351): the controller and the tool runner pass on one +// reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's resolver +// keep three, the first two minutes after the send; the bus is kept as it was. + +// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. +func withTheGatesTimes(t *testing.T) { + t.Helper() + wasSettle, wasEvery, wasBound, wasWindow := gateSettle, gateEvery, gateBound, gateWitnessWindow + wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles + gateSettle, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second + gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false + t.Cleanup(func() { + gateSettle, gateEvery, gateBound, gateWitnessWindow = wasSettle, wasEvery, wasBound, wasWindow + gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles + }) +} + +func TestEachModuleHasItsGateRule(t *testing.T) { + withTheGatesTimes(t) + for _, c := range []struct { + module string + claims []catalogue.Claim + passes int + settle time.Duration + witnessed bool + said string + }{ + {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, + {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, + {catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"}, + {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, + {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"app", nil, 1, 0, false, "a leaf module's rule"}, + } { + r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) + if r.passes != c.passes || r.settle != c.settle || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { + t.Errorf("%s: %+v; want %d readings, settle %s, witnessed %v, said %q", c.module, r, c.passes, c.settle, + c.witnessed, c.said) + } + } +} + +// Every module that was core stays core for its delivery budget and its walk: only its readings change. +func TestTheCoreStaysTheCore(t *testing.T) { + for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { + if !coreModule(m, catalogue.Manifest{Module: m}) { + t.Errorf("%s is no longer judged as the core for its class", m) + } + } + if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { + t.Error("the resolver's holder is no longer judged as the core for its class") + } +} + +// gateOn judges one module on one machine, healthy at every reading. +type gateOn struct { + module, node string + g *inventory.PlanGate + holder *lease.Holder +} + +func aGateOn(module, node string, since time.Time) *gateOn { + return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, + Applied: map[string]time.Time{}}} +} + +func (o *gateOn) read(at time.Time, h health, why string) { + r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module}) + floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) + tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, + worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) +} + +func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(15 * time.Second) + o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} + + // Ready, but inside the witness window, which opened before it took the lease: not counted. + for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { + o.read(sent.Add(after), healthGood, "") + if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { + t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) + } + } + o.read(sent.Add(80*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { + t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) + } + if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || + !strings.Contains(o.g.Why, "ADR 0351") { + t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) + } +} + +// A controller that starts again takes the lease again: its witness window starts again, and so does the +// gate's wait. +func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(10 * time.Second) + o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} + o.read(sent.Add(90*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) + } + o.read(sent.Add(130*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed { + t.Fatalf("not passed after the new window: %+v", o.g) + } +} + +// Without the lease's word (a controller carried by another gate), the window counts from the applied report, +// and from the send at the least. +func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + r := gateRuleOf("mesh-controller", catalogue.Manifest{}) + if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { + t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) + } + if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, + map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { + t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) + } +} + +// A controller that a witness put back, or that never becomes ready, fails its gate as before. +func TestAControllerThatFailsIsStillFailed(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") + if o.g.Verdict != inventory.GateFailed { + t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) + } + o = aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != "" { + t.Fatalf("failed before the bound: %+v", o.g) + } + o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { + t.Fatalf("never ready: %+v; want failed at the bound", o.g) + } +} + +// The node-engine and the resolver keep three readings, the first at least two minutes after the send. +func TestTheNodeEngineAndTheResolverKeepThreeReadings(t *testing.T) { + withTheGatesTimes(t) + for _, module := range []string{"mesh-host", "dnsmasq"} { + sent := time.Now() + o := aGateOn(module, "ace", sent) + rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) + floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) + judge := func(after time.Duration) { + tallyGate(o.g, gateJudging{modules: []string{module}, reading: map[string]health{module: healthGood}, + worst: healthGood, rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, + sent.Add(after)) + } + for _, after := range []time.Duration{20 * time.Second, 60 * time.Second, 100 * time.Second} { + judge(after) + } + if o.g.Passes != 0 || o.g.Verdict != "" { + t.Fatalf("%s: readings before two minutes counted: %+v", module, o.g) + } + judge(2 * time.Minute) + judge(160 * time.Second) + if o.g.Verdict != "" || o.g.Passes != 2 { + t.Fatalf("%s: after two readings %+v; want still judging", module, o.g) + } + judge(200 * time.Second) + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 3 || o.g.Wants != 3 || + !strings.Contains(o.g.Why, "the core's rule: 3 readings") { + t.Fatalf("%s: after three readings %+v; want passed by the core's rule", module, o.g) + } + } +} + +// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. +func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("node-tools", "ace", sent) + o.g.Applied["ace"] = sent.Add(10 * time.Second) + o.read(sent.Add(30*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) + } + o.read(sent.Add(70*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { + t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) + } + + o = aGateOn(catalogue.DeliverySeat, "novox", sent) + o.read(sent.Add(5*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { + t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) + } +} + +// A send carrying the controller and the node-engine is judged by the node-engine's three readings. +func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) + } + for _, after := range []time.Duration{70 * time.Second, 2 * time.Minute, 160 * time.Second} { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, + "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) + } + if g.Verdict != "" || g.Wants != 3 || g.Healthy["mesh-controller"] != 3 { + t.Fatalf("%+v; want the send still judging by three readings, the controller's own counted", g) + } +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index ced2ccce..06294a82 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -125,6 +125,10 @@ func aGateMesh(t *testing.T) *gateMesh { t.Cleanup(func() { gatherGateFacts = wasGather }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) @@ -285,31 +289,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { } } -// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three -// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3). -func TestAModuleIsJudgedByItsClass(t *testing.T) { - for _, c := range []struct { - module string - claims []catalogue.Claim - passes int - settles bool - }{ - {"app", nil, 1, false}, - {"nzbget", nil, 1, false}, - {"mesh-controller", nil, 3, true}, - {"mesh-host", nil, 3, true}, - {"node-tools", nil, 3, true}, - {"nats", nil, 3, true}, - {catalogue.DeliverySeat, nil, 3, true}, - {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true}, - } { - p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if p != c.passes || s != c.settles { - t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles) - } - } -} - // A first machine that never becomes healthy fails its gate at the bound, and is put back. func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { judgeLeavesAsTheCore(t) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 0560cdd8..263a9f01 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -134,6 +134,10 @@ func aBacklog(t *testing.T) *backlogMesh { return f, nil } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 09e212d0..4fa3b172 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -113,6 +113,10 @@ func TestReplay318(t *testing.T) { wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 138f0180..5a107b23 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -249,6 +249,10 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { } t.Cleanup(func() { sendRollout = wasSend }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, 0 t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 48c34f2e..bf3ad9a4 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -272,6 +272,10 @@ func aSettledMesh(t *testing.T) *walkMesh { return f, nil } wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index c91a8fc0..df003116 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -131,6 +131,10 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh { t.Cleanup(func() { gatherGateFacts = wasGather }) wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) @@ -440,15 +444,15 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } } -// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node -// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three -// readings. -func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { +// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 +// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool +// runner's rule: one reading, not before the node-engine's witness window for it has ended. +func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { tm := aTierMesh(t, broker.RuntimeModule, "app1") ctx := t.Context() // One reading counted per advance, as every test with a gate left open does: a gate judged with no time // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateSettle = time.Hour, time.Hour + gateEvery, gateWitnessWindow = time.Hour, time.Hour for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -457,10 +461,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { if lead == nil { lead = p.Modules["app1"].Gate } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses { - t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { + t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) } - gateEvery, gateSettle = 0, 0 + gateEvery, gateWitnessWindow = 0, 0 for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -469,8 +473,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) } for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) { - t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses) + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || + !strings.Contains(g.Why, "the tool runner's rule")) { + t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) } } } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index c17344a0..abc02bf6 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -323,11 +323,16 @@ type PlanGate struct { // machine's report is held against. Absent on a gate kept before it was, which reads the report // against the send made last, as before. Sent map[string]SentDeclaration `json:"sent,omitempty"` + // Applied is, per machine, the time of the first report the gate read of its send applied: the + // node-engine started the build before it, so a witness window counted from it ends no earlier than the + // node-engine's own (novox/hq ADR 0351 decisions 1 and 4). + Applied map[string]time.Time `json:"applied,omitempty"` // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282 - // decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three. + // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, one for any + // other module (novox/hq ADR 0351, changing ADR 0282 decision 4 and ADR 0304 decision 3). Zero in a gate + // judged before it was said: three. Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` // Last is what the newest judging found wanting, while it still may pass.