novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

Give every module of a failed send a verdict, and excuse only the wait a build's own send brought (hq issue 318 review)

Browse Source
This commit is contained in:
jochen
2026-10-08 15:25:58 +02:00
parent a759ac65a5
commit 363898ec8a
10 changed files with 599 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/module_health.go
+118 -33
View File
@@ -152,14 +152,23 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
sort.Strings(modules)
seen := map[string]bool{}
// became is, per module, the kind of condition this statement says of it: what a standing one of the
// other kind turned into.
became := map[string]string{}
heldOn := map[string]string{}
providers := map[catalogue.Chosen]bool{}
for _, m := range modules {
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
// operator, never urgent, cleared on the first statement that no longer says it.
if said, waits := personWait(m, node, unhealthy[m]); waits {
o := reloginObservation(m, node, said, unhealthy[m])
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
// consumers are held under it, and its own condition is where they are said.
if hold != nil {
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
}
seen[o.Key()] = true
became[m] = kindReloginNeeded
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
continue
}
@@ -176,14 +185,10 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
providers[p] = true
continue
}
if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
}
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
}
seen[o.Key()] = true
became[m] = kindModuleUnhealthy
c, isOpen := standing[o.Key()]
if streaks[m] < moduleUnhealthyAfter && !isOpen {
continue // unconfirmed: one statement can be wrong; `node show` lists it
@@ -207,7 +212,18 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
if on, held := heldOn[key]; held {
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
}
if _, err := k.Clear(ctx, key, why); err != nil {
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
// line says what it became.
resolved := ""
switch {
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
}
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
problems = append(problems, err.Error())
}
}
@@ -229,7 +245,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
var raisedAt *conditions.Condition
for i, c := range hold.open {
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
raisedAt = &hold.open[i]
}
}
@@ -246,35 +262,97 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
return nil
}
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
if waiters := hold.waitersOn(p); len(waiters) > 0 {
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
if said, waits := personWait(p.Module, p.Node, rs); waits {
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
}
if o.Key() != raisedAt.Key {
return nil // its own statement says it next
}
sayWaitingOn(&o, hold.waitersOn(p))
_, err := k.Observe(ctx, o)
return err
}
// reloginKey is a module's relogin-needed condition on a machine.
func reloginKey(module, node string) string {
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
}
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
if inv == nil {
return ""
}
n, err := inv.NodeByName(ctx, node)
if err != nil {
return ""
}
return n.Account
}
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
// person in again, and a restart of the module's service would start it in the same session.
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
// person in again, and a restart of the module's service would start it in the same session. operator is
// the machine's operator account, when known: the words say "your account" only for it.
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
o := moduleUnhealthyObservation(module, node, rs)
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
w := reloginWords(module, node)
accounts := waitingAccounts(module, rs)
yours := operator != "" && len(accounts) > 0
for _, a := range accounts {
yours = yours && a == operator
}
w := reloginWords(module, node, yours)
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
return o
}
// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
func reloginWords(module, node string) words {
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
func reloginWords(module, node string, yours bool) words {
if yours {
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
"cannot run until you log in again. Its update is in place and nothing was undone.",
conditions.Capital(module), module),
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
}
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
conditions.Capital(module), node, module),
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
}
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
seen := map[string]bool{}
var out []string
for _, r := range rs {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
seen[accountOf(r)] = true
out = append(out, accountOf(r))
}
}
sort.Strings(out)
return out
}
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
// waits on it, whether it is not working or waits for a new login.
func sayWaitingOn(o *conditions.Observation, waiters []string) {
if len(waiters) == 0 {
return
}
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
}
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
@@ -352,18 +430,14 @@ const kindReloginNeeded = "relogin-needed"
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
// resource still starting is not unhealthy and does not make a wait either. Pure.
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
waiting := map[string]bool{}
var accounts []string
for _, r := range rs {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
waiting[accountOf(r)] = true
accounts = append(accounts, accountOf(r))
}
}
accounts := waitingAccounts(module, rs)
if len(accounts) == 0 {
return "", false
}
waiting := map[string]bool{}
for _, a := range accounts {
waiting[a] = true
}
var units []string
for _, r := range rs {
if r.Module != module || r.State != link.StateUnhealthy {
@@ -377,7 +451,6 @@ func personWait(module, machine string, rs []inventory.ResourceHealth) (string,
return "", false
}
}
sort.Strings(accounts)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
"running session began before it was; log out of every session and in again, or reboot", machine, module,
strings.Join(accounts, ", "))
@@ -415,6 +488,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
}
wait, waits := personWait(module, machine, h.Resources)
// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
// account already waiting before this send was not brought to it by this build, and a later build of
// the module sent while the login is still owed is judged as before. Two seconds of skew between the
// machine's clock and the controller's, as the witness allows.
if waits {
for _, r := range h.Resources {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
waits = false
}
}
}
for _, r := range h.Resources {
if r.Module != module {
continue