novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318)

Browse Source
This commit is contained in:
jochen
2026-10-08 14:20:55 +02:00
parent 6df30b6d98
commit e3b5c224e8
8 changed files with 694 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/module_health.go
+100 -2
View File
@@ -73,7 +73,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
for _, r := range h.Resources {
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
Check: r.Check, Needs: r.Needs}
Check: r.Check, Needs: r.Needs, Account: r.Account}
resources = append(resources, kept)
if r.State == link.StateUnhealthy && r.Module != "" {
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
@@ -135,7 +135,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
standing := map[string]conditions.Condition{}
for _, c := range open {
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded) && c.Subject.Machine == node {
standing[c.Key] = c
}
}
@@ -155,6 +155,19 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
heldOn := map[string]string{}
providers := map[catalogue.Chosen]bool{}
for _, m := range modules {
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
// operator, never urgent, cleared on the first statement that no longer says it.
if said, waits := personWait(m, node, unhealthy[m]); waits {
o := reloginObservation(m, node, said, unhealthy[m])
seen[o.Key()] = true
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
continue
}
if _, err := k.Observe(ctx, o); err != nil {
problems = append(problems, err.Error())
}
continue
}
o := moduleUnhealthyObservation(m, node, unhealthy[m])
if hold != nil {
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
@@ -188,6 +201,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
module := strings.TrimSuffix(c.Subject.ID, "."+node)
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
if c.Kind == kindReloginNeeded {
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
}
if on, held := heldOn[key]; held {
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
}
@@ -239,6 +255,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
return err
}
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence.
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
o := moduleUnhealthyObservation(module, node, rs)
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
return o
}
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
@@ -296,9 +320,76 @@ func orNotSaid(s string) string {
return s
}
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
const kindReloginNeeded = "relogin-needed"
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
// excused:
//
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
// running began before;
// - every other unhealthy resource of the module runs in the own service manager of one of those very
// accounts (Account names it): the manager that began before the group and does not hold it.
//
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
// another account's manager, a resource whose engine does not say whose manager it is in, an account
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
// resource still starting is not unhealthy and does not make a wait either. Pure.
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
waiting := map[string]bool{}
var accounts []string
for _, r := range rs {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
waiting[accountOf(r)] = true
accounts = append(accounts, accountOf(r))
}
}
if len(accounts) == 0 {
return "", false
}
var units []string
for _, r := range rs {
if r.Module != module || r.State != link.StateUnhealthy {
continue
}
switch {
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
units = append(units, r.Target)
default:
return "", false
}
}
sort.Strings(accounts)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
"running session began before it was; log out of every session and in again, or reboot", machine, module,
strings.Join(accounts, ", "))
if len(units) > 0 {
sort.Strings(units)
said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
}
return said, true
}
// accountOf is the account a resource of kind account is: named by the engine, or its target.
func accountOf(r inventory.ResourceHealth) string {
if r.Account != "" {
return r.Account
}
return r.Target
}
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
//
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
// and not something a machine can meet within a bound.
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
if f.healthErr != nil {
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
@@ -310,10 +401,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
if h.HeardAt.Before(since) {
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
}
wait, waits := personWait(module, machine, h.Resources)
for _, r := range h.Resources {
if r.Module != module {
continue
}
if waits && r.State == link.StateUnhealthy {
continue
}
switch r.State {
case link.StateHealthy:
case link.StateStarting:
@@ -329,6 +424,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
reasonAfter(r.Reason))
}
}
if waits {
return healthPerson, wait
}
return healthGood, ""
}