novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

No build reaches a machine without a gate; a release plan walks what waits (hq ADR 0236)

Browse Source 
A send carries the machine's whole declaration, so at the switch to roll the next send of
anything would have carried the old default's backlog, unjudged, to every machine. A gated
send now carries and judges everything waiting on its machine; every other send is refused
or leaves the machine; a release plan walks what waits one machine at a time, the control
node last, and one that fails holds the next until a person releases it.
This commit is contained in:
jochen
2026-10-06 19:14:25 +02:00
parent 41f7b2c152
commit 2bfa6ae4a0
15 changed files with 1163 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/gate.go
+41 -6
View File
@@ -139,7 +139,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
} else {
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
}
if component == lease.ComponentController {
if theLease != nil {
h, found, err := theLease.holder(ctx)
switch {
case err != nil:
@@ -147,6 +147,9 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
case found:
f.holder = &h
}
if component != lease.ComponentController && f.holderErr != nil {
f.holderErr = nil // read only for the controller's own judging
}
}
return f, nil
}
@@ -263,12 +266,31 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
To: state.Commit, Since: &start}
state.Gate = g
}
pairs := []judged{}
for _, n := range g.Machines {
pairs = append(pairs, judged{module: module, node: n})
}
return judgeMoves(ctx, open, g, pairs, now)
}
// judged is one module on one machine, as a gate judges it.
type judged struct{ module, node string }
// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and
// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is
// wanting, and which modules), or the verdict. Answers the verdict once there is one.
func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) {
if g.Verdict != "" {
return g.Verdict, nil
}
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
return "", nil
}
for _, c := range g.Carried {
if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) {
pairs = append(pairs, judged{module: c.Module, node: c.Node})
}
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
@@ -278,8 +300,16 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
return "", err
}
worst, why := healthGood, ""
for _, n := range g.Machines {
h, said := judgeHealth(module, g.Component, shelf[module], n, *g.Since, facts)
var failing []string
broken := map[string]bool{}
for _, j := range pairs {
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, facts)
if h != healthGood && !slices.Contains(failing, j.module) {
failing = append(failing, j.module)
}
if h == healthBroken {
broken[j.module] = true
}
if h > worst {
worst, why = h, said
} else if h == worst && h != healthGood && why == "" {
@@ -288,15 +318,17 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
}
switch {
case worst == healthBroken:
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
g.Failing = failing
decide(g, inventory.GateFailed, why, now)
case worst == healthNotYet:
g.Passes, g.LastPass, g.Last = 0, nil, why
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
if now.Sub(*g.Since) > gateBound {
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
}
default:
g.Passes++
g.LastPass, g.Last = &now, ""
g.LastPass, g.Last, g.Failing = &now, "", nil
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second)), now)
@@ -320,6 +352,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str
if err != nil && state.Build != "" {
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
}
passCarried(ctx, open, p, g, module)
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
strings.Join(g.Machines, ", "), g.Why)
if module == catalogue.ControllerSeatName {
@@ -443,7 +476,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
}
sent, err := sendRollout(ctx, open, g.Machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
if err != nil {
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -553,6 +586,8 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
out = append(out, witnessObservation(node, r))
}
}
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
out = append(out, backlogObservation()...)
return sortedFound(dedupeObservations(out)), nil
}