Archived
jochen
eba36f02af
The gate judged a module by what the mesh saw from outside, so a container that crash-looped after it applied passed it. Each machine's node-engine now states the health of every long-running resource it runs; the controller keeps the newest statement per machine, raises module.<module>.<machine>.unhealthy on the second statement in a row, clears it on the first that does not say it, and the gate passes a module only when every long-running resource of it is stated healthy since the send. An engine that states nothing is judged as before.
250 lines
9.2 KiB
Go
250 lines
9.2 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
|
// Phase A).
|
|
//
|
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
|
// machine states, in every report and as an event between reports, the state of every long-running
|
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
|
//
|
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
|
|
|
// The condition a module's health raises.
|
|
const (
|
|
kindModuleUnhealthy = "module-unhealthy"
|
|
// sourceHealth is what raised it: the machine's own statement.
|
|
sourceHealth = "health"
|
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
|
moduleUnhealthyAfter = 2
|
|
)
|
|
|
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
|
var healthRefused atomic.Int64
|
|
|
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
|
type moduleHealth struct {
|
|
inv *inventory.Inventory
|
|
keeper func() *conditions.Keeper
|
|
}
|
|
|
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
|
}
|
|
|
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
|
// older statement than the one kept is refused, by when the engine looked.
|
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
|
now time.Time) error {
|
|
if h.Contract == 0 {
|
|
return nil
|
|
}
|
|
prev, had, err := inv.HealthOf(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if had && h.At.Before(prev.SaidAt) {
|
|
healthRefused.Add(1)
|
|
return nil
|
|
}
|
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
|
for _, r := range h.Resources {
|
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts}
|
|
resources = append(resources, kept)
|
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
|
}
|
|
}
|
|
streaks := map[string]int{}
|
|
for module := range unhealthy {
|
|
streaks[module] = prev.Streaks[module] + 1
|
|
}
|
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
|
HeardAt: now, Resources: resources, Streaks: streaks})
|
|
if err != nil || !stored {
|
|
if err == nil {
|
|
healthRefused.Add(1)
|
|
}
|
|
return err
|
|
}
|
|
if k == nil {
|
|
return nil
|
|
}
|
|
return judgeModuleHealth(ctx, k, node, unhealthy, streaks, now)
|
|
}
|
|
|
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
|
// statement no longer says.
|
|
func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
|
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
standing := map[string]conditions.Condition{}
|
|
for _, c := range open {
|
|
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
|
|
standing[c.Key] = c
|
|
}
|
|
}
|
|
var problems []string
|
|
modules := make([]string, 0, len(unhealthy))
|
|
for m := range unhealthy {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
seen := map[string]bool{}
|
|
for _, m := range modules {
|
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
|
seen[o.Key()] = true
|
|
c, isOpen := standing[o.Key()]
|
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
|
}
|
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
|
o.Severity = conditions.Urgent
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
for key, c := range standing {
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
|
if _, err := k.Clear(ctx, key, fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
var words, said []string
|
|
for _, r := range rs {
|
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
|
Said: strings.Join(said, "; ")}
|
|
}
|
|
|
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
|
func reasonWords(r inventory.ResourceHealth) string {
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
|
case "down":
|
|
return "is not running"
|
|
case "":
|
|
return "is unhealthy"
|
|
}
|
|
return "is unhealthy: " + r.Reason
|
|
}
|
|
|
|
func orNotSaid(s string) string {
|
|
if s == "" {
|
|
return "no reason said"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
|
if f.healthErr != nil {
|
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
|
}
|
|
h, states := f.health[machine]
|
|
if !states {
|
|
return healthGood, ""
|
|
}
|
|
if h.HeardAt.Before(since) {
|
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
|
}
|
|
for _, r := range h.Resources {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateStarting:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
|
case link.StateUnhealthy:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
|
default:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
|
reasonAfter(r.Reason))
|
|
}
|
|
}
|
|
return healthGood, ""
|
|
}
|
|
|
|
func reasonAfter(s string) string {
|
|
if s == "" {
|
|
return ""
|
|
}
|
|
return ": " + s
|
|
}
|
|
|
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
|
// since when, an unhealthy one said once marked unconfirmed.
|
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|
if !had {
|
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
|
}
|
|
if len(h.Resources) == 0 {
|
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
|
}
|
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
|
for _, r := range h.Resources {
|
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
|
r.Since.Local().Format("2006-01-02 15:04"))
|
|
if r.Reason != "" {
|
|
line += " — " + r.Reason
|
|
}
|
|
if r.Restarts > 0 {
|
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
|
}
|
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
|
line += " (unconfirmed: said once)"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
return out
|
|
}
|