diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index 3fff9be3..cf8eba68 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -81,9 +81,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { } return f, err } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) - gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { advancePlans(ctx, b.open) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index f5aa1689..05a2d4d0 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -28,9 +28,12 @@ import ( // that machine reported it applied. A build that applies and then does nothing, crashes, serves no // tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by // the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the -// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then -// are the rest sent. +// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third +// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the +// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states +// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 +// decision 3). +// Only then are the rest sent. // // **A failing gate stops the plan and puts the previous build back there.** The build is marked failed // at its gate — registration refuses it and nothing sends it again on its own — the module's registered @@ -41,22 +44,30 @@ import ( // The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. var ( - // gateSettle is how long the first machine must stay healthy, at the least. - gateSettle = 2 * time.Minute + // gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the + // gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR + // 0352). Readings gateEvery apart, the first at the apply, put three inside it. + gateWithin = 2 * time.Minute // gateBound is how long after the apply the build may take to become healthy. gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the - // least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every - // resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304 - // decision 3 changed it, issue 524). + // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at + // the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the + // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 + // decision 4 as ADR 0304 decision 3 changed it, issue 524). gatePasses = 3 gateLeafPasses = 1 gateLeafSettles = false gateEvery = 40 * time.Second + // gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its + // start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either + // counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR + // 0351 decisions 1 and 4). + gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin) ) // coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's -// own path, or one holding the mesh's resolver. Every other module is a leaf. +// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's +// delivery budget; how many readings its gate takes is gateRuleOf's. func coreModule(module string, m catalogue.Manifest) bool { if _, own := onTheControllersPath[module]; own { return true @@ -69,12 +80,72 @@ func coreModule(module string, m catalogue.Manifest) bool { return coreComponent(module) != "" } -// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first. -func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) { - if coreModule(module, m) { - return gatePasses, true +// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the +// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it; +// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading +// (novox/hq ADR 0352). said names the rule in the gate's text. +type gateRule struct { + passes int + within time.Duration + witnessed bool + said string +} + +// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its +// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send, +// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never +// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner, +// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4); +// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304 +// decision 3). +func gateRuleOf(module string, m catalogue.Manifest) gateRule { + switch { + case coreComponent(module) == lease.ComponentController: + return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case coreComponent(module) == lease.ComponentNodeTools: + return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " + + "after the node-engine's witness window (novox/hq ADR 0351)"} + case module == catalogue.DeliverySeat: + return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} + case coreModule(module, m): + return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+ + "or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)} } - return gateLeafPasses, gateLeafSettles + r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} + if gateLeafSettles { + r.within = gateWithin + r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send", + gateLeafPasses, gateWithin) + } + return r +} + +// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the +// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A +// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it +// applied (applied, as the gate first read it) and, for the controller, before the new controller took the +// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. +// A controller that starts again takes the lease again, and so moves its floor on. Pure. +func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, + holder *lease.Holder) time.Time { + floor := since + if !rule.witnessed { + return floor + } + for _, n := range machines { + start := since + if at, read := applied[n]; read && at.After(start) { + start = at + } + if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) { + start = holder.Taken + } + if end := start.Add(gateWitnessWindow); end.After(floor) { + floor = end + } + } + return floor } // gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions @@ -501,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri return judgeMoves(ctx, open, g, pairs, now) } +// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one +// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352). +func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool { + most := g.Since.Add(gateWithin) + return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most) +} + // judged is one module on one machine, as a gate judges it. type judged struct{ module, node string } @@ -511,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if g.Verdict != "" { return g.Verdict, nil } - if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery { + // A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing: + // a module healthy throughout is never held past it (novox/hq ADR 0352). + if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) { return "", nil } for _, c := range g.Carried { @@ -608,6 +688,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs why = said } } + // **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read + // each machine report its send applied, and who holds the controller lease, the earliest its reading counts. + if g.Applied == nil { + g.Applied = map[string]time.Time{} + } + for node := range byMachine { + r, said := facts.reports[node] + if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied && + facts.reportedOn(node, r) { + g.Applied[node] = *r.At + } + } + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + var on []string + for _, j := range pairs { + if j.module == m { + on = append(on, j.node) + } + } + rules[m] = gateRuleOf(m, shelf[m]) + floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder) + } + tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why, + rules: rules, floors: floors}, now) + return g.Verdict, nil +} + +// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading +// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's +// rule and the earliest its reading counts. +type gateJudging struct { + modules []string + reading map[string]health + waits map[string]string + worst health + why string + rules map[string]gateRule + floors map[string]time.Time +} + +// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but +// for the gate it writes. +func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { + modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why if g.Healthy == nil { g.Healthy = map[string]int{} } @@ -623,38 +748,53 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs delete(g.HealthyAt, m) continue } - // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more - // often while another module beside it is not yet healthy counts no faster. - if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { - g.Healthy[m]++ - g.HealthyAt[m] = now - } if w := waits[m]; w != "" { if g.Waits == nil { g.Waits = map[string]string{} } g.Waits[m] = w } + // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the + // controller's and the tool runner's after the witness window, any other module's from the send. + if now.Before(j.floors[m]) { + continue + } + // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more + // often while another module beside it is not yet healthy counts no faster. + if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { + g.Healthy[m]++ + g.HealthyAt[m] = now + } } - settled := now.Sub(*g.Since) >= gateSettle - // What the send needs is what its most demanding module needs: a send carrying a core module is judged - // as the core. - wants, settles := 0, false + // What the send needs is what its most demanding module needs — the most readings, then the latest + // floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout + // its two minutes (novox/hq ADR 0352). + wants, floor, within, said := 0, time.Time{}, time.Duration(0), "" for _, m := range modules { - p, s := gateWants(m, shelf[m]) - wants, settles = max(wants, p), settles || s + r, f := j.rules[m], j.floors[m] + if r.passes > wants || (r.passes == wants && f.After(floor)) { + wants, within, said = r.passes, r.within, r.said + } + if f.After(floor) { + floor = f + } } if len(modules) == 0 { // A send judged on no machine of its modules (its first machine not running them) has no class to // be judged by: it is judged as the core, never passed at once without a reading. - wants, settles = gatePasses, true + wants, floor, within = gatePasses, *g.Since, gateWithin + said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin) + } + // heldLongEnough is a rule's most: within after the send, healthy at every reading since it. + heldLongEnough := func(within time.Duration) bool { + return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within)) } g.Wants = wants - // A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after + // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). passedAlone := func(m string) bool { - p, s := gateWants(m, shelf[m]) - return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p + r := j.rules[m] + return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within))) } // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. fail := func(why string) { @@ -671,6 +811,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs pastBound := now.Sub(*g.Since) > gateBound switch { case worst == healthBroken: + g.Faulted = true g.Read(now, false, g.BrokenWhy) var judging []string for _, m := range modules { @@ -689,25 +830,36 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — // the provider's own condition says what is wrong (ADR 0240 rule 5). + g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) case worst == healthNotYet: + g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing g.Read(now, false, why) if pastBound { fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) } + case now.Before(floor): + // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). + g.HealthySeen = true + // The bound still holds: a module that cannot be read healthy after its floor fails at it. + g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", + floor.UTC().Format(time.RFC3339), said), nil + if pastBound { + fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last)) + } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. + g.HealthySeen = true g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil g.Read(now, true, "") - if g.Passes >= wants && (settled || !settles) { - decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, - now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) + if g.Passes >= wants || heldLongEnough(within) { + decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, + now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) } } - return g.Verdict, nil } // judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index 35929eb5..b8377d77 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -21,11 +21,15 @@ import ( // wait, and a provider waiting for a login says who waits on it. // withGateBounds sets the gate's bounds for one test. -func withGateBounds(t *testing.T, settle, every, bound time.Duration) { +func withGateBounds(t *testing.T, within, every, bound time.Duration) { t.Helper() - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound - gateSettle, gateEvery, gateBound = settle, every, bound - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery, gateBound = within, every, bound + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) } // factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says @@ -56,7 +60,7 @@ func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() - withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) + withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second) since := time.Now().Add(-time.Second) base := factsOn(t, "laptop", since, healthyContainer("app")) wasGather := gatherGateFacts @@ -97,7 +101,7 @@ const broker_runtime = "node-tools" func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { open := aMesh(t) ctx := t.Context() - withGateBounds(t, 0, time.Hour, time.Hour) + withGateBounds(t, time.Hour, time.Hour, time.Hour) since := time.Now().Add(-time.Minute) base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) @@ -376,7 +380,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}} return f, nil } - withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second) + withGateBounds(t, time.Hour, 0, 10*time.Second) release := func() inventory.Plan { t.Helper() diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go new file mode 100644 index 00000000..06777685 --- /dev/null +++ b/cmd/mesh-controller/gate_rule_test.go @@ -0,0 +1,338 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass +// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's +// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so. + +// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. +func withTheGatesTimes(t *testing.T) { + t.Helper() + wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow + wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles + gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second + gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false + t.Cleanup(func() { + gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow + gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles + }) +} + +func TestEachModuleHasItsGateRule(t *testing.T) { + withTheGatesTimes(t) + for _, c := range []struct { + module string + claims []catalogue.Claim + passes int + within time.Duration + witnessed bool + said string + }{ + {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, + {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, + {catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"}, + {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, + {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, + {"app", nil, 1, 0, false, "a leaf module's rule"}, + } { + r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) + if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { + t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within, + c.witnessed, c.said) + } + } +} + +// Every module that was core stays core for its delivery budget and its walk: only its readings change. +func TestTheCoreStaysTheCore(t *testing.T) { + for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { + if !coreModule(m, catalogue.Manifest{Module: m}) { + t.Errorf("%s is no longer judged as the core for its class", m) + } + } + if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { + t.Error("the resolver's holder is no longer judged as the core for its class") + } +} + +// gateOn judges one module on one machine, healthy at every reading. +type gateOn struct { + module, node string + g *inventory.PlanGate + holder *lease.Holder +} + +func aGateOn(module, node string, since time.Time) *gateOn { + return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, + Applied: map[string]time.Time{}}} +} + +func (o *gateOn) read(at time.Time, h health, why string) { + r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module}) + floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) + tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, + worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) +} + +func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(15 * time.Second) + o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} + + // Ready, but inside the witness window, which opened before it took the lease: not counted. + for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { + o.read(sent.Add(after), healthGood, "") + if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { + t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) + } + } + o.read(sent.Add(80*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { + t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) + } + if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || + !strings.Contains(o.g.Why, "ADR 0351") { + t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) + } +} + +// A controller that starts again takes the lease again: its witness window starts again, and so does the +// gate's wait. +func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.g.Applied["novox"] = sent.Add(10 * time.Second) + o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} + o.read(sent.Add(90*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) + } + o.read(sent.Add(130*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed { + t.Fatalf("not passed after the new window: %+v", o.g) + } +} + +// Without the lease's word (a controller carried by another gate), the window counts from the applied report, +// and from the send at the least. +func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + r := gateRuleOf("mesh-controller", catalogue.Manifest{}) + if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { + t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) + } + if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, + map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { + t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) + } +} + +// A controller that a witness put back, or that never becomes ready, fails its gate as before. +func TestAControllerThatFailsIsStillFailed(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") + if o.g.Verdict != inventory.GateFailed { + t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) + } + o = aGateOn("mesh-controller", "novox", sent) + o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != "" { + t.Fatalf("failed before the bound: %+v", o.g) + } + o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") + if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { + t.Fatalf("never ready: %+v; want failed at the bound", o.g) + } +} + +// threeReadings judges a module under the three-reading rule on one machine, the send at sent. +func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) { + t.Helper() + g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent} + rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) + floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) + return g, func(after time.Duration, h health) { + why := "" + if h != healthGood { + why = "the node-engine reports another build" + } + tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why, + rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after)) + } +} + +// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two +// minutes (novox/hq ADR 0352). +func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) { + withTheGatesTimes(t) + for _, module := range []string{"mesh-host", "dnsmasq"} { + g, judge := threeReadings(t, module, time.Now()) + judge(15*time.Second, healthGood) + judge(55*time.Second, healthGood) + if g.Verdict != "" || g.Passes != 2 { + t.Fatalf("%s: after two readings %+v; want still judging", module, g) + } + judge(95*time.Second, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 || + !strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") { + t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g) + } + } +} + +// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at +// two minutes comes sooner than the spacing after the last pass. +func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + last := sent.Add(100 * time.Second) + held := &inventory.PlanGate{Since: &sent, LastPass: &last} + if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) { + t.Fatal("the judging at two minutes is not due, or is due before") + } + held.Faulted = true + if dueAtTheMost(held, sent.Add(2*time.Minute)) { + t.Fatal("a send that was not healthy at a reading is judged at two minutes") + } + + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(100*time.Second, healthGood) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 2 { + t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 1 { + t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g) + } +} + +// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. +func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(10*time.Second, healthNotYet) + judge(100*time.Second, healthGood) + judge(2*time.Minute, healthGood) + if g.Verdict != inventory.GatePassed || g.Faulted { + t.Fatalf("%+v; want passed at two minutes", g) + } +} + +// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one +// never healthy fails at the bound and is put back; one found broken fails. +func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) { + withTheGatesTimes(t) + g, judge := threeReadings(t, "mesh-host", time.Now()) + judge(20*time.Second, healthNotYet) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthGood) + judge(2*time.Minute+10*time.Second, healthGood) + if g.Verdict != inventory.GatePassed || g.Passes != 3 { + t.Fatalf("%+v; want passed only on its third healthy reading", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthNotYet) + judge(2*time.Minute, healthGood) + if g.Verdict != "" || !g.Faulted { + t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g) + } + judge(10*time.Minute+time.Second, healthNotYet) + if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") { + t.Fatalf("never healthy again: %+v; want failed at the bound", g) + } + g, judge = threeReadings(t, "mesh-host", time.Now()) + judge(60*time.Second, healthGood) + judge(100*time.Second, healthBroken) + if g.Verdict != inventory.GateFailed { + t.Fatalf("a broken reading: %+v; want failed, to be put back", g) + } +} + +// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. +func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + o := aGateOn("node-tools", "ace", sent) + o.g.Applied["ace"] = sent.Add(10 * time.Second) + o.read(sent.Add(30*time.Second), healthGood, "") + if o.g.Verdict != "" { + t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) + } + o.read(sent.Add(70*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { + t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) + } + + o = aGateOn(catalogue.DeliverySeat, "novox", sent) + o.read(sent.Add(5*time.Second), healthGood, "") + if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { + t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) + } +} + +// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at +// two minutes (novox/hq ADR 0352 decision 3). +func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + } + floors["mesh-host"] = sent + floors["mesh-controller"] = sent.Add(90 * time.Second) + judge := func(after time.Duration, h health) { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h}, + worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after)) + } + judge(30*time.Second, healthGood) + if g.Passes != 0 { + t.Fatalf("a reading before the controller's floor was counted: %+v", g) + } + judge(70*time.Second, healthNotYet) + judge(2*time.Minute, healthGood) + if !g.Faulted || g.Verdict != "" { + t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g) + } +} + +// A send carrying the controller and the node-engine is judged by the node-engine's three readings. +func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { + withTheGatesTimes(t) + sent := time.Now() + g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} + modules := []string{"mesh-controller", "mesh-host"} + rules, floors := map[string]gateRule{}, map[string]time.Time{} + for _, m := range modules { + rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) + floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) + } + for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} { + tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, + "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) + } + if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 { + t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g) + } +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index ced2ccce..d8062644 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -124,10 +124,14 @@ func aGateMesh(t *testing.T) *gateMesh { } t.Cleanup(func() { gatherGateFacts = wasGather }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateSettle, gateEvery = 0, time.Hour - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", @@ -272,7 +276,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { g := aGateMesh(t) ctx := t.Context() - gateSettle = time.Hour // a core module could not pass for an hour + gateWithin = time.Hour // a core module could not pass on fewer readings for an hour advancePlans(ctx, g.open) // the first machine is sent the new build advancePlans(ctx, g.open) // one healthy judging p := g.plan(t) @@ -285,31 +289,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { } } -// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three -// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3). -func TestAModuleIsJudgedByItsClass(t *testing.T) { - for _, c := range []struct { - module string - claims []catalogue.Claim - passes int - settles bool - }{ - {"app", nil, 1, false}, - {"nzbget", nil, 1, false}, - {"mesh-controller", nil, 3, true}, - {"mesh-host", nil, 3, true}, - {"node-tools", nil, 3, true}, - {"nats", nil, 3, true}, - {catalogue.DeliverySeat, nil, 3, true}, - {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true}, - } { - p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if p != c.passes || s != c.settles { - t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles) - } - } -} - // A first machine that never becomes healthy fails its gate at the bound, and is put back. func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { judgeLeavesAsTheCore(t) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 0560cdd8..35b7d35c 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -133,11 +133,15 @@ func aBacklog(t *testing.T) *backlogMesh { } return f, nil } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound - gateSettle, gateEvery = 0, time.Hour + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return b } diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 09e212d0..53fb53ca 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -112,10 +112,14 @@ func TestReplay318(t *testing.T) { } wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) @@ -184,7 +188,7 @@ func TestReplay318(t *testing.T) { return f, nil } // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. - gateSettle, gateEvery, gateBound = 0, 0, 0 + gateWithin, gateEvery, gateBound = time.Hour, 0, 0 for i := 0; i < 12; i++ { advancePlans(ctx, open) diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 138f0180..2798542d 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -248,9 +248,13 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { return names, nil } t.Cleanup(func() { sendRollout = wasSend }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound - gateSettle, gateEvery = 0, 0 - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, 0 + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 48c34f2e..45df338a 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -271,12 +271,16 @@ func aSettledMesh(t *testing.T) *walkMesh { } return f, nil } - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound - gateSettle, gateEvery = 0, time.Hour + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather settingsDataMoves = wasData - gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return w } diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index c91a8fc0..97241324 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -130,10 +130,14 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh { } t.Cleanup(func() { gatherGateFacts = wasGather }) - wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateSettle, gateEvery = 0, time.Hour - t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) built := time.Now().UTC() states := map[string]*inventory.PlanModule{} @@ -222,6 +226,9 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) ctx := t.Context() inv := tm.open.inventory + // The node tools are held inside their witness window, which the node-engine keeps while they settle: their + // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). + gateWitnessWindow = time.Hour advancePlans(ctx, tm.open) if len(tm.sent) != 1 { @@ -405,7 +412,10 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } return f, err } - gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second + // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the + // judgings that find the node tools broken, and is judged to its own pass after them (the hold the + // settle time gave before novox/hq ADR 0352 counted readings from the send). + gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second for i := 0; i < 20 && len(seen) < 2; i++ { advancePlans(ctx, tm.open) } @@ -440,15 +450,15 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { } } -// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node -// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three -// readings. -func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { +// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 +// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool +// runner's rule: one reading, not before the node-engine's witness window for it has ended. +func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { tm := aTierMesh(t, broker.RuntimeModule, "app1") ctx := t.Context() // One reading counted per advance, as every test with a gate left open does: a gate judged with no time // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateSettle = time.Hour, time.Hour + gateEvery, gateWitnessWindow = time.Hour, time.Hour for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -457,10 +467,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { if lead == nil { lead = p.Modules["app1"].Gate } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses { - t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { + t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) } - gateEvery, gateSettle = 0, 0 + gateEvery, gateWitnessWindow = 0, 0 for i := 0; i < 6; i++ { advancePlans(ctx, tm.open) } @@ -469,8 +479,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) { t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) } for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) { - t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses) + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || + !strings.Contains(g.Why, "the tool runner's rule")) { + t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) } } } diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index c17344a0..5f058270 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -323,13 +323,25 @@ type PlanGate struct { // machine's report is held against. Absent on a gate kept before it was, which reads the report // against the send made last, as before. Sent map[string]SentDeclaration `json:"sent,omitempty"` + // Applied is, per machine, the time of the first report the gate read of its send applied: the + // node-engine started the build before it, so a witness window counted from it ends no earlier than the + // node-engine's own (novox/hq ADR 0351 decisions 1 and 4). + Applied map[string]time.Time `json:"applied,omitempty"` // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that // does not resets them. Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282 - // decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three. + // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two + // minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR + // 0282 decision 4 and ADR 0304 decision 3). Zero in a gate + // judged before it was said: three. Wants int `json:"wants,omitempty"` LastPass *time.Time `json:"last_pass,omitempty"` + // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading + // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). + Faulted bool `json:"faulted,omitempty"` + // HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor + // counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3). + HealthySeen bool `json:"healthy_seen,omitempty"` // Last is what the newest judging found wanting, while it still may pass. Last string `json:"last,omitempty"` // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.