novox/mesh-controller
8
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Merge origin/main into feat/335, and pin mesh-host at #98's new head

Browse Source
This commit is contained in:
jschoubben
2026-10-14 00:01:21 +02:00
parent b1546ed28a 72214ae989
commit a69c0e50dc
159 changed files with 11270 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/gate.go
+188 -36
View File
@@ -28,9 +28,12 @@ import (
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
// are the rest sent.
// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
// decision 3).
// Only then are the rest sent.
//
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -41,22 +44,30 @@ import (
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
var (
// gateSettle is how long the first machine must stay healthy, at the least.
gateSettle = 2 * time.Minute
// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
gateWithin = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
// decision 3 changed it, issue 524).
// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
// decision 4 as ADR 0304 decision 3 changed it, issue 524).
gatePasses = 3
gateLeafPasses = 1
gateLeafSettles = false
gateEvery = 40 * time.Second
// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
// 0351 decisions 1 and 4).
gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
)
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
// own path, or one holding the mesh's resolver. Every other module is a leaf.
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
// delivery budget; how many readings its gate takes is gateRuleOf's.
func coreModule(module string, m catalogue.Manifest) bool {
if _, own := onTheControllersPath[module]; own {
return true
@@ -69,12 +80,72 @@ func coreModule(module string, m catalogue.Manifest) bool {
return coreComponent(module) != ""
}
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
if coreModule(module, m) {
return gatePasses, true
// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
// (novox/hq ADR 0352). said names the rule in the gate's text.
type gateRule struct {
passes int
within time.Duration
witnessed bool
said string
}
// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
// decision 3).
func gateRuleOf(module string, m catalogue.Manifest) gateRule {
switch {
case coreComponent(module) == lease.ComponentController:
return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
"after the node-engine's witness window (novox/hq ADR 0351)"}
case coreComponent(module) == lease.ComponentNodeTools:
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
"after the node-engine's witness window (novox/hq ADR 0351)"}
case module == catalogue.DeliverySeat:
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
case coreModule(module, m):
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
}
return gateLeafPasses, gateLeafSettles
r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
if gateLeafSettles {
r.within = gateWithin
r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
gateLeafPasses, gateWithin)
}
return r
}
// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
// A controller that starts again takes the lease again, and so moves its floor on. Pure.
func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
holder *lease.Holder) time.Time {
floor := since
if !rule.witnessed {
return floor
}
for _, n := range machines {
start := since
if at, read := applied[n]; read && at.After(start) {
start = at
}
if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
start = holder.Taken
}
if end := start.Add(gateWitnessWindow); end.After(floor) {
floor = end
}
}
return floor
}
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
@@ -501,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
return judgeMoves(ctx, open, g, pairs, now)
}
// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
most := g.Since.Add(gateWithin)
return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
}
// judged is one module on one machine, as a gate judges it.
type judged struct{ module, node string }
@@ -511,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
if g.Verdict != "" {
return g.Verdict, nil
}
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
// a module healthy throughout is never held past it (novox/hq ADR 0352).
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
return "", nil
}
for _, c := range g.Carried {
@@ -616,6 +696,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
why = said
}
}
// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
if g.Applied == nil {
g.Applied = map[string]time.Time{}
}
for node := range byMachine {
r, said := facts.reports[node]
if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
facts.reportedOn(node, r) {
g.Applied[node] = *r.At
}
}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
var on []string
for _, j := range pairs {
if j.module == m {
on = append(on, j.node)
}
}
rules[m] = gateRuleOf(m, shelf[m])
floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
}
tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
rules: rules, floors: floors}, now)
return g.Verdict, nil
}
// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
// rule and the earliest its reading counts.
type gateJudging struct {
modules []string
reading map[string]health
waits map[string]string
worst health
why string
rules map[string]gateRule
floors map[string]time.Time
}
// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
// for the gate it writes.
func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
if g.Healthy == nil {
g.Healthy = map[string]int{}
}
@@ -631,38 +756,53 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
delete(g.HealthyAt, m)
continue
}
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
// often while another module beside it is not yet healthy counts no faster.
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
g.Healthy[m]++
g.HealthyAt[m] = now
}
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
}
// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
// controller's and the tool runner's after the witness window, any other module's from the send.
if now.Before(j.floors[m]) {
continue
}
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
// often while another module beside it is not yet healthy counts no faster.
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
g.Healthy[m]++
g.HealthyAt[m] = now
}
}
settled := now.Sub(*g.Since) >= gateSettle
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
// as the core.
wants, settles := 0, false
// What the send needs is what its most demanding module needs — the most readings, then the latest
// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
// its two minutes (novox/hq ADR 0352).
wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
for _, m := range modules {
p, s := gateWants(m, shelf[m])
wants, settles = max(wants, p), settles || s
r, f := j.rules[m], j.floors[m]
if r.passes > wants || (r.passes == wants && f.After(floor)) {
wants, within, said = r.passes, r.within, r.said
}
if f.After(floor) {
floor = f
}
}
if len(modules) == 0 {
// A send judged on no machine of its modules (its first machine not running them) has no class to
// be judged by: it is judged as the core, never passed at once without a reading.
wants, settles = gatePasses, true
wants, floor, within = gatePasses, *g.Since, gateWithin
said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
}
// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
heldLongEnough := func(within time.Duration) bool {
return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
}
g.Wants = wants
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
passedAlone := func(m string) bool {
p, s := gateWants(m, shelf[m])
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
r := j.rules[m]
return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
}
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
fail := func(why string) {
@@ -679,6 +819,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
pastBound := now.Sub(*g.Since) > gateBound
switch {
case worst == healthBroken:
g.Faulted = true
g.Read(now, false, g.BrokenWhy)
var judging []string
for _, m := range modules {
@@ -697,25 +838,36 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
case worst == healthWaiting:
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
// the provider's own condition says what is wrong (ADR 0240 rule 5).
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
g.Read(now, false, why)
case worst == healthNotYet:
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
g.Read(now, false, why)
if pastBound {
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
}
case now.Before(floor):
// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
g.HealthySeen = true
// The bound still holds: a module that cannot be read healthy after its floor fails at it.
g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
floor.UTC().Format(time.RFC3339), said), nil
if pastBound {
fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
}
default:
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
g.HealthySeen = true
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
g.Read(now, true, "")
if g.Passes >= wants && (settled || !settles) {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
if g.Passes >= wants || heldLongEnough(within) {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
}
}
return g.Verdict, nil
}
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each