novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Delete the controller's plan machinery: plans, batches, merge window, tiers, gate, retry and healers H1 and H2 (release 6, issue 1945 B7)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 16:27:55 +02:00
parent 1b0fd8b442
commit dc456d5660
236 changed files with 572 additions and 44491 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/signals.go
-195
View File
@@ -58,8 +58,6 @@ const (
controlNodeUrgentAfter = 30 * time.Minute
// reportAtLeast is the least a machine is given to report a send (S2).
reportAtLeast = 2 * time.Minute
// tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at.
tierAtLeast = planWaitBound
// loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4).
loopDeafAfter = 2 * time.Minute
// callDefault is the bound of a verb that declares none (S7); push's and build's are longer.
@@ -102,20 +100,6 @@ var signalsTable = []signalRow{
newest: func(f *signalFacts) time.Time {
return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard })
}},
{Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent",
Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep",
Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports,
newest: func(f *signalFacts) time.Time {
return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt })
}},
{Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered",
Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional",
Kind: "stalled", Severity: conditions.Warning, Phase: 1,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans,
newest: func(f *signalFacts) time.Time {
return newestOf(f.plans, func(p planFacts) time.Time { return p.entered })
}},
{Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller",
Trigger: "while its consumers have pending messages", Bound: "2 min",
Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1,
@@ -189,58 +173,6 @@ var signalsTable = []signalRow{
newest: func(f *signalFacts) time.Time {
return newestOf(f.handActs, func(a link.HandAct) time.Time { return a.At })
}},
{Row: "S16", Signal: "a walk waiting for its delivery's word is let go", Emitter: "mesh-delivery, through deliver",
Trigger: "each merge whose walk waits (novox/hq ADR 0239)",
Bound: "30 min, then urgent after 4 h — raised by the controller whatever mesh-delivery says of itself, " +
"naming `plans go` as the way on",
Kind: kindWalkWaiting, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchWaits,
newest: func(f *signalFacts) time.Time {
return newestOf(f.waits, func(w waitFacts) time.Time { return w.since })
}},
{Row: "S18", Signal: "a batch of merges is cut into its walk", Emitter: "controller's merge window",
Trigger: "each batch (novox/hq ADR 0276)",
Bound: "a minute past merge-window-at-most, while no walk is open: the controller failed to cut it",
Kind: kindBatchNotCut, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesNotCut,
newest: func(f *signalFacts) time.Time {
return newestOf(f.batches, func(b batchFacts) time.Time { return b.atMost })
}},
{Row: "S19", Signal: "a batch waiting behind an open walk is cut when it ends", Emitter: "controller's merge window",
Trigger: "each batch closed while a walk is open (novox/hq ADR 0276)",
Bound: "the walk's bound, a tier's bound for each of its tiers, from when the batch closed: naming the walk",
Kind: kindBatchBehindWalk, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesBehind,
newest: func(f *signalFacts) time.Time {
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
}},
{Row: "S22", Signal: "the last member of a delivery group merges", Emitter: "mesh-delivery's group, held by the controller's merge window",
Trigger: "each batch held for a group with a member still to merge (novox/hq ADR 0418)",
Bound: "merge-window-at-most from the group's first merge: naming the member and why it did not merge; " +
"nothing of the group is delivered while it waits",
Kind: kindGroupNotMerged, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.plansErr }, watch: watchGroupsNotMerged,
newest: func(f *signalFacts) time.Time {
return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
}},
{Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan",
Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)",
Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " +
"words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " +
"from the plan's last change, and ended when the plan is no longer failed",
Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed,
newest: func(f *signalFacts) time.Time {
return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
}},
{Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator",
Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)",
Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " +
"machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " +
"pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.",
Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld,
newest: func(f *signalFacts) time.Time { return time.Time{} }},
{Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names",
Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)",
Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " +
@@ -249,34 +181,6 @@ var signalsTable = []signalRow{
Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages,
newest: func(f *signalFacts) time.Time { return time.Time{} }},
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
"machine applied; cleared an hour after the last",
Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2,
needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals,
newest: func(f *signalFacts) time.Time {
return newestOf(f.refusedSends, func(s inventory.Send) time.Time {
if s.RefusedAt == nil {
return time.Time{}
}
return *s.RefusedAt
})
}},
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
"since when and the mesh-controller.bus call; cleared once the bus's machine has been sent the build the mesh holds",
Kind: kindBusStepWaiting, Severity: conditions.Warning, Phase: 3,
needs: func(f *signalFacts) error {
if f.plansErr != nil {
return f.plansErr
}
return f.busErr
}, watch: watchBusWaits,
newest: func(f *signalFacts) time.Time {
return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
}},
{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " +
"is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " +
@@ -393,105 +297,6 @@ func watchReports(f *signalFacts) []conditions.Observation {
return out
}
func watchPlans(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, p := range f.plans {
// Held only by the bus's planned step (S17, novox/hq issue 336): a wait
// for a person, said as itself, not a walk running late.
if p.bus {
continue
}
in := f.now.Sub(p.entered)
if in <= p.bound {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway",
Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+
"usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))),
Resolved: "Delivery of " + repoName(p.repository) + " moves again",
Severity: conditions.Warning,
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
ago(p.bound), p.waiting),
Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)})
}
return out
}
// kindPlanFailed is S23's kind: plan.<id>.plan-failed.
const kindPlanFailed = "plan-failed"
// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no
// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to
// the operator). A plan failed longer ago than this is left to `plans`.
const planFailedWithin = 7 * 24 * time.Hour
// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when
// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2).
// The condition ends when the plan is no longer failed, because it is no longer among the facts.
func watchPlansFailed(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, p := range f.failed {
if f.now.Sub(p.since) > planFailedWithin {
continue
}
headline, explanation, needs, actions, resolved := planFailedWords(p)
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed,
Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit),
p.failureSaid()),
Said: p.failureSaid() + p.refusalSaid(),
Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved})
}
return out
}
// kindWalkWaiting is S16's kind: plan.<id>.waiting.
const kindWalkWaiting = "waiting"
// watchWaits is S16: a walk that has waited for its delivery's word past its bound, said by the controller
// itself — a delivery's owner that is up and never says go (a bug, a delivery stuck in its own table) would
// otherwise leave a merge waiting for ever with nothing open (novox/hq ADR 0239).
func watchWaits(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, w := range f.waits {
in := f.now.Sub(w.since)
if in <= waitBound {
continue
}
severity := conditions.Warning
if in > waitUrgentAfter {
severity = conditions.Urgent
}
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.id, Kind: kindWalkWaiting,
Severity: severity,
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
"deliveries that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
short(w.commit), ago(in), w.awaits, mergesWords(w), w.id),
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: walkWaitingWords(w, in, severity),
Needs: waitingNeeds(severity),
Actions: waitingActions(w.id, severity),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
}
return out
}
// mergesWords is the merges a waiting walk answers (novox/hq ADR 0276): every delivery it carries, not one
// commit that a supersession may already have ended (issue 362). Its own commit for a walk naming none.
func mergesWords(w waitFacts) string {
if len(w.merges) == 0 {
return short(w.commit)
}
var out []string
for _, m := range w.merges {
out = append(out, m.Repository+"@"+short(m.Commit))
}
return readableList(out)
}
func watchLoop(f *signalFacts) []conditions.Observation {
if f.loop.pending == 0 {
return nil