Judge how a module says it is ready, beside whether it stays up (hq ADR 0240, to-be 48 Phase B)
Liveness alone could not see a web application whose port was open and whose program ran while every request hung for eleven hours (issue 145). A resource now carries the `health` its module declared: the engine makes http and tcp looks itself from the machine to the endpoint's published port, reads a unit's readiness from the show it already makes, hands an exec command or the image's own check to the runtime as the container's check with the declared timing and reads its state from the inspect it already makes, and asks a module's tool on its own node tools. Starting until the check passed, unhealthy once its looks after the grace fail the declared number of times; never more looks than the measured budget; nothing restarted. The statement says contract 2, which tells the controller this engine may be sent the field.
This commit is contained in:
jochen
@@ -81,6 +81,9 @@ type Resource struct {
|
||||
// Scope and User are a service's manager: "user" and the account for a unit in an account's own.
|
||||
Scope string `json:"scope,omitempty"`
|
||||
User string `json:"user,omitempty"`
|
||||
// Check is how it is ready, as its module declared (ADR 0240 rule 2, Phase B); nil judges it alive
|
||||
// or not, and nothing more.
|
||||
Check *declaration.Health `json:"check,omitempty"`
|
||||
}
|
||||
|
||||
// LongRunning is every long-running resource a declaration asks this machine to run for a module: its
|
||||
@@ -102,12 +105,12 @@ func LongRunning(d *declaration.Declaration, held map[string]bool) []Resource {
|
||||
if v.RunOnce || v.Schedule != "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, Resource{Module: module, ID: v.ID, Kind: KindContainer, Target: v.Name})
|
||||
out = append(out, Resource{Module: module, ID: v.ID, Kind: KindContainer, Target: v.Name, Check: v.Health})
|
||||
case *declaration.Service:
|
||||
if v.State != "running" {
|
||||
continue
|
||||
}
|
||||
res := Resource{Module: module, ID: v.ID, Kind: KindService, Target: v.Unit}
|
||||
res := Resource{Module: module, ID: v.ID, Kind: KindService, Target: v.Unit, Check: v.Health}
|
||||
if v.UserScoped() {
|
||||
res.Scope, res.User = declaration.ScopeUser, v.User
|
||||
}
|
||||
@@ -116,7 +119,8 @@ func LongRunning(d *declaration.Declaration, held map[string]bool) []Resource {
|
||||
if v.RunOnce || v.Schedule != "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, Resource{Module: module, ID: v.ID, Kind: KindProcess, Target: v.Name + ".service"})
|
||||
out = append(out, Resource{Module: module, ID: v.ID, Kind: KindProcess, Target: v.Name + ".service",
|
||||
Check: v.Health})
|
||||
}
|
||||
}
|
||||
return out
|
||||
@@ -150,6 +154,10 @@ type Observed struct {
|
||||
// Started is when the runtime says the current run started, as it says it: a change with no restart
|
||||
// counted is a restart somebody made, which is a new start.
|
||||
Started string
|
||||
// Health is the runtime's own word on the check it runs as the container's — healthy, unhealthy,
|
||||
// starting — empty when the container carries none; HealthSaid what its last look printed.
|
||||
Health string
|
||||
HealthSaid string
|
||||
}
|
||||
|
||||
// Runtime is what one look reads: every container's state in one read, and every unit's per manager.
|
||||
@@ -169,6 +177,21 @@ type State struct {
|
||||
Restarts int `json:"restarts,omitempty"`
|
||||
}
|
||||
|
||||
// CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
|
||||
func (s State) CheckOf() string {
|
||||
if s.Check == nil {
|
||||
return ""
|
||||
}
|
||||
return s.Check.Kind
|
||||
}
|
||||
|
||||
func (s State) NeedsOf() string {
|
||||
if s.Check == nil {
|
||||
return ""
|
||||
}
|
||||
return s.Check.Needs
|
||||
}
|
||||
|
||||
// Statement is one look at every long-running resource: when, and each resource's state.
|
||||
type Statement struct {
|
||||
At time.Time
|
||||
@@ -206,6 +229,15 @@ type kept struct {
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Streak int `json:"streak,omitempty"`
|
||||
|
||||
// Readiness since the current start (Phase B): whether the declared check has passed, how many of
|
||||
// its looks after the grace failed in a row, and what the last failing one said.
|
||||
Passed bool `json:"passed,omitempty"`
|
||||
Failing int `json:"failing,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// probing says a look of the engine's own is under way; due when the next is.
|
||||
probing bool
|
||||
due time.Time
|
||||
}
|
||||
|
||||
// file is the judge's file beside the node's state.
|
||||
@@ -233,13 +265,20 @@ type Judge struct {
|
||||
dirty bool
|
||||
// said is the statement said last, to know a change by.
|
||||
said map[string]string
|
||||
|
||||
// Probes make the looks the engine makes itself — http, tcp and a module's tool (Phase B). Nil
|
||||
// makes them from this machine.
|
||||
Probes *Probes
|
||||
// Budget is the most looks a minute every declared check together may cost (ADR 0240: never more
|
||||
// than measured on the busiest machine); over it the engine's own looks are spaced out.
|
||||
Budget int
|
||||
}
|
||||
|
||||
// Open is the judge whose file is at path, reading what it kept. A file that cannot be read is said and
|
||||
// started afresh: a count lost is a crash loop judged from now, never a machine left unjudged.
|
||||
func Open(path string, rt Runtime) (*Judge, error) {
|
||||
j := &Judge{path: path, runtime: rt, Now: time.Now, Grace: DefaultGrace, Settle: SettleWindow,
|
||||
f: file{Kept: map[string]*kept{}}, said: map[string]string{}}
|
||||
f: file{Kept: map[string]*kept{}}, said: map[string]string{}, Budget: Budget}
|
||||
raw, err := os.ReadFile(path)
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
@@ -277,6 +316,10 @@ func (j *Judge) Set(resources []Resource) {
|
||||
// The same id now names another thing: judged as a new one, its count kept.
|
||||
counted := k.Counted
|
||||
j.f.Kept[r.ID] = &kept{Resource: r, Counted: counted}
|
||||
} else if ok && !sameCheck(k.Check, r.Check) {
|
||||
// Its check changed — another kind, another port, another timing: what the old one found
|
||||
// says nothing about the new, which looks again at once.
|
||||
k.Check, k.Passed, k.Failing, k.Why, k.due = r.Check, false, 0, "", time.Time{}
|
||||
}
|
||||
}
|
||||
for id := range j.f.Kept {
|
||||
@@ -314,7 +357,7 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
||||
o := observed[keyOf(r)]
|
||||
j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
|
||||
seen[r.ID] = true
|
||||
st.Resources = append(st.Resources, State{Resource: r, State: k.State, Reason: k.Reason, Since: k.Since,
|
||||
st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
|
||||
Streak: k.Streak, Restarts: k.Counted})
|
||||
word := k.State + "/" + k.Reason
|
||||
if j.said[r.ID] != word {
|
||||
@@ -348,6 +391,8 @@ func (j *Judge) judge(k *kept, o Observed, now time.Time, held, blind bool, why
|
||||
fresh := func(o Observed, started time.Time) {
|
||||
k.Seen, k.Identity, k.RuntimeRestarts, k.RuntimeStarted = o.Found, o.Identity, o.Restarts, o.Started
|
||||
k.Started, k.Recent = started, nil
|
||||
// Every start is judged ready afresh (to-be 48 §4).
|
||||
k.Passed, k.Failing, k.Why, k.due = false, 0, "", time.Time{}
|
||||
}
|
||||
|
||||
// **Held is neither alive nor dead**, and the window ending is a start: judged from a fresh grace.
|
||||
@@ -407,23 +452,34 @@ func (j *Judge) judge(k *kept, o Observed, now time.Time, held, blind bool, why
|
||||
}
|
||||
k.Recent = recent
|
||||
|
||||
inGrace := now.Before(k.Started.Add(j.Grace))
|
||||
inGrace := now.Before(k.Started.Add(j.graceOf(k.Resource)))
|
||||
switch {
|
||||
case len(k.Recent) >= 2:
|
||||
set(Unhealthy, ReasonRestarting)
|
||||
case inGrace:
|
||||
set(Starting, "")
|
||||
case !o.Found || !o.Running:
|
||||
case !inGrace && (!o.Found || !o.Running):
|
||||
if o.Restarting {
|
||||
set(Unhealthy, ReasonRestarting)
|
||||
} else {
|
||||
set(Unhealthy, ReasonDown)
|
||||
}
|
||||
case k.Check != nil:
|
||||
// Alive, or still in its grace: how ready it is is the declared check's to say.
|
||||
set(ready(k, o, inGrace))
|
||||
case inGrace:
|
||||
set(Starting, "")
|
||||
default:
|
||||
set(Healthy, "")
|
||||
}
|
||||
}
|
||||
|
||||
// graceOf is a resource's grace: its declared one, or the default (to-be 48 §1).
|
||||
func (j *Judge) graceOf(r Resource) time.Duration {
|
||||
if r.Check != nil {
|
||||
return r.Check.GraceOf()
|
||||
}
|
||||
return j.Grace
|
||||
}
|
||||
|
||||
// read is one read of everything: every container at once, every unit per manager. unread names each
|
||||
// group that could not be read, with why.
|
||||
func (j *Judge) read(ctx context.Context) (map[string]Observed, map[string]string) {
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
package liveness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// Readiness, declared (novox/hq ADR 0240 rules 2 and 3, to-be 48 §2–§3, Phase B).
|
||||
//
|
||||
// A long-running resource that declares `health` is judged alive as before, and then ready by its check:
|
||||
// **starting** while in its grace and not yet passed, **healthy** once it passed, **unhealthy** once its
|
||||
// declared number of looks after the grace failed in a row — with what the check found, in words.
|
||||
//
|
||||
// **The engine runs every kind where it is cheapest, and owns every verdict:**
|
||||
//
|
||||
// - http and tcp it makes itself, from the machine to the port the endpoint is published on — the path
|
||||
// a caller takes (issue 145), and no execution inside the container;
|
||||
// - unit it reads from the show of the service manager every look already makes;
|
||||
// - exec and runtime the runtime runs as the container's own check, with the declared timing (its
|
||||
// retries are the failing looks, its start period the grace), and the engine reads the state from the
|
||||
// inspect every look already makes;
|
||||
// - tool it asks of the module's tool on this machine's own node tools.
|
||||
//
|
||||
// **Never more looks than the measured budget.** The busiest machine runs about ninety looks a minute at
|
||||
// the default interval (research 032 §7); when the declared checks together would cost more, the engine
|
||||
// spaces its own looks out until they do not, and says so.
|
||||
|
||||
// Budget is the most looks a minute the declared checks on one machine may cost together: one check per
|
||||
// long-running resource at the default interval on the busiest machine (research 032 §7).
|
||||
const Budget = 90
|
||||
|
||||
// ProbesAtOnce is how many of the engine's own looks run at the same time.
|
||||
const ProbesAtOnce = 8
|
||||
|
||||
// ready is a resource's readiness, on a look where it is alive or still in its grace: starting until its
|
||||
// check passed, healthy once it has, unhealthy once its failing looks after the grace reach the declared
|
||||
// number. The runtime's and the unit's checks are read from what the look itself read; the engine's own
|
||||
// looks were recorded by Probe.
|
||||
func ready(k *kept, o Observed, inGrace bool) (string, string) {
|
||||
c := k.Check
|
||||
switch c.Kind {
|
||||
case declaration.HealthRuntime, declaration.HealthExec:
|
||||
switch o.Health {
|
||||
case Healthy:
|
||||
k.Passed, k.Failing, k.Why = true, 0, ""
|
||||
case Unhealthy:
|
||||
// The runtime's retries are the declared failing looks: unhealthy is already that many.
|
||||
if !inGrace {
|
||||
k.Failing, k.Why = c.Looks, orSaid(o.HealthSaid, "its check failed")
|
||||
}
|
||||
case "":
|
||||
if !inGrace {
|
||||
why := "the container carries no check: it was made before its check was declared"
|
||||
if c.Kind == declaration.HealthRuntime {
|
||||
why = "its image ships no check to adopt"
|
||||
}
|
||||
k.Failing, k.Why = c.Looks, why
|
||||
}
|
||||
}
|
||||
case declaration.HealthUnit:
|
||||
// Active and not failed — and for a unit that notifies, notified, which the manager says by
|
||||
// being active only once it was.
|
||||
if o.Running {
|
||||
k.Passed, k.Failing, k.Why = true, 0, ""
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !inGrace && k.Failing >= c.Looks:
|
||||
return Unhealthy, c.Words() + ": " + orSaid(k.Why, "it failed")
|
||||
case k.Passed:
|
||||
return Healthy, ""
|
||||
case k.Why != "":
|
||||
return Starting, "not ready yet — " + c.Words() + ": " + k.Why
|
||||
}
|
||||
return Starting, ""
|
||||
}
|
||||
|
||||
func orSaid(s, otherwise string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return otherwise
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// sameCheck says two declarations of a check are the same check.
|
||||
func sameCheck(a, b *declaration.Health) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
}
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
// ownLook says whether a check is one the engine makes itself.
|
||||
func ownLook(c *declaration.Health) bool {
|
||||
return c != nil && (c.Kind == declaration.HealthHTTP || c.Kind == declaration.HealthTCP || c.Kind == declaration.HealthTool)
|
||||
}
|
||||
|
||||
// Spacing is how much the engine's own looks are spaced out so every declared check together costs no
|
||||
// more than the budget: 1 when they fit, more when they would not.
|
||||
func (j *Judge) Spacing() float64 {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.spacing()
|
||||
}
|
||||
|
||||
func (j *Judge) spacing() float64 {
|
||||
perMinute := 0.0
|
||||
for _, r := range j.f.Resources {
|
||||
if r.Check == nil || r.Check.Kind == declaration.HealthUnit {
|
||||
continue // a unit's readiness rides on the show every look already makes
|
||||
}
|
||||
if every := r.Check.Every(); every > 0 {
|
||||
perMinute += float64(time.Minute) / float64(every)
|
||||
}
|
||||
}
|
||||
budget := j.Budget
|
||||
if budget <= 0 {
|
||||
budget = Budget
|
||||
}
|
||||
if perMinute <= float64(budget) {
|
||||
return 1
|
||||
}
|
||||
return perMinute / float64(budget)
|
||||
}
|
||||
|
||||
// Probe makes the engine's own looks — http, tcp, a module's tool — each at its interval, spaced out to
|
||||
// the budget, never two of one resource at once and never more than ProbesAtOnce together, until ctx
|
||||
// ends. A held resource is not looked at. It reads; it never acts (ADR 0240 rule 6).
|
||||
func (j *Judge) Probe(ctx context.Context) {
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
slots := make(chan struct{}, ProbesAtOnce)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
for _, d := range j.due() {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
go func(d dueLook) {
|
||||
defer func() { <-slots }()
|
||||
ok, why := j.probes().Look(ctx, d.module, d.check)
|
||||
j.Record(d.id, d.started, ok, why)
|
||||
}(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dueLook is one of the engine's own looks to make now.
|
||||
type dueLook struct {
|
||||
id, module string
|
||||
check *declaration.Health
|
||||
started time.Time
|
||||
}
|
||||
|
||||
// due is every look of the engine's own whose time has come, marked under way.
|
||||
func (j *Judge) due() []dueLook {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
now := j.Now()
|
||||
spacing := j.spacing()
|
||||
var out []dueLook
|
||||
for _, r := range j.f.Resources {
|
||||
k := j.f.Kept[r.ID]
|
||||
if k == nil || !ownLook(r.Check) || k.probing || k.State == Held || k.Started.IsZero() || now.Before(k.due) {
|
||||
continue
|
||||
}
|
||||
k.probing = true
|
||||
k.due = now.Add(time.Duration(float64(r.Check.Every()) * spacing))
|
||||
out = append(out, dueLook{id: r.ID, module: r.Module, check: r.Check, started: k.Started})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Record keeps what one of the engine's own looks found, for the next look to fold in: a pass is a pass
|
||||
// whenever it came; a failure counts only after the grace. A look begun before the resource's current
|
||||
// start says nothing about it.
|
||||
func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
k := j.f.Kept[id]
|
||||
if k == nil {
|
||||
return
|
||||
}
|
||||
k.probing = false
|
||||
if !k.Started.Equal(started) || k.Check == nil {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
k.Passed, k.Failing, k.Why = true, 0, ""
|
||||
case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))):
|
||||
k.Why = why
|
||||
default:
|
||||
k.Failing++
|
||||
k.Why = why
|
||||
}
|
||||
j.dirty = true
|
||||
}
|
||||
|
||||
func (j *Judge) probes() *Probes {
|
||||
if j.Probes != nil {
|
||||
return j.Probes
|
||||
}
|
||||
return &Probes{}
|
||||
}
|
||||
|
||||
// Probes make the looks the engine makes itself (to-be 48 §3).
|
||||
type Probes struct {
|
||||
// Host is where an endpoint's port is dialled: this machine, by default its loopback.
|
||||
Host string
|
||||
// AskTool asks a module's tool on this machine's own node tools; nil says it cannot be asked.
|
||||
AskTool func(ctx context.Context, module, tool string) (healthy bool, why string, err error)
|
||||
}
|
||||
|
||||
// Look makes one look of a check, and says whether it passed and, when not, what it found.
|
||||
func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string) {
|
||||
timeout := c.Within()
|
||||
if timeout <= 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
switch c.Kind {
|
||||
case declaration.HealthHTTP:
|
||||
return p.http(ctx, c, timeout)
|
||||
case declaration.HealthTCP:
|
||||
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", p.address(c.Port))
|
||||
if err != nil {
|
||||
return false, dialWords(err, timeout)
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true, ""
|
||||
case declaration.HealthTool:
|
||||
if p.AskTool == nil {
|
||||
return false, "its tool cannot be asked: this engine has no link to the node tools"
|
||||
}
|
||||
healthy, why, err := p.AskTool(ctx, module, c.Tool)
|
||||
switch {
|
||||
case err != nil:
|
||||
return false, "asking it: " + firstLine(err.Error())
|
||||
case !healthy:
|
||||
return false, orSaid(why, "it answered not healthy, and not why")
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
return false, "the engine does not make a " + c.Kind + " look itself"
|
||||
}
|
||||
|
||||
func (p *Probes) address(port int) string {
|
||||
host := p.Host
|
||||
if host == "" {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
return net.JoinHostPort(host, strconv.Itoa(port))
|
||||
}
|
||||
|
||||
// http is one request: the path, on the port, expecting the declared status (any under 400 when none
|
||||
// was declared) and, when declared, a text in the answer. A redirect is an answer, not followed: a login
|
||||
// page elsewhere says nothing about this program.
|
||||
func (p *Probes) http(ctx context.Context, c *declaration.Health, timeout time.Duration) (bool, string) {
|
||||
scheme := c.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
url := scheme + "://" + p.address(c.Port) + c.Path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
req.Header.Set("User-Agent", "mesh-node-engine health")
|
||||
client := &http.Client{
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
// Whether it answers, not whom to trust: a program's own certificate is its own business.
|
||||
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true},
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, dialWords(err, timeout)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
switch {
|
||||
case c.Status != 0 && res.StatusCode != c.Status:
|
||||
return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, c.Status)
|
||||
case c.Status == 0 && res.StatusCode >= 400:
|
||||
return false, fmt.Sprintf("answered %d", res.StatusCode)
|
||||
}
|
||||
if c.Body != "" {
|
||||
body, err := io.ReadAll(io.LimitReader(res.Body, 64<<10))
|
||||
if err != nil {
|
||||
return false, "its answer could not be read: " + firstLine(err.Error())
|
||||
}
|
||||
if !strings.Contains(string(body), c.Body) {
|
||||
return false, fmt.Sprintf("answered %d without %q", res.StatusCode, c.Body)
|
||||
}
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// dialWords is a failed look in words: no answer in time, refused, or what the system said.
|
||||
func dialWords(err error, timeout time.Duration) string {
|
||||
var asked *url.Error
|
||||
if errors.As(err, &asked) {
|
||||
err = asked.Err // what happened, without the address it happened at
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF):
|
||||
return "it closed the connection without answering"
|
||||
case errors.Is(err, context.DeadlineExceeded) || errors.Is(err, os.ErrDeadlineExceeded):
|
||||
return fmt.Sprintf("no answer within %s", timeout)
|
||||
case strings.Contains(err.Error(), "connection refused"):
|
||||
return "connection refused"
|
||||
case strings.Contains(err.Error(), "Client.Timeout") || strings.Contains(err.Error(), "timeout"):
|
||||
return fmt.Sprintf("no answer within %s", timeout)
|
||||
}
|
||||
return firstLine(err.Error())
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package liveness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// Readiness, declared (novox/hq ADR 0240 rules 2 and 3, Phase B): starting until its check passed, healthy
|
||||
// once it has, unhealthy once its failing looks after the grace reach the declared number; an http check
|
||||
// dials the endpoint's current port after a port change; the runtime's check is read, never run by the
|
||||
// engine; and never more looks than the budget.
|
||||
|
||||
func httpCheck(port int) *declaration.Health {
|
||||
return &declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: port, Path: "/healthz",
|
||||
Interval: "10s", Timeout: "2s", Looks: 2, Grace: "30s"}
|
||||
}
|
||||
|
||||
func portOf(t *testing.T, url string) int {
|
||||
t.Helper()
|
||||
_, p, err := net.SplitHostPort(strings.TrimPrefix(url, "http://"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, _ := strconv.Atoi(p)
|
||||
return n
|
||||
}
|
||||
|
||||
// lookOnce makes the engine's own due looks at once and waits for them, as Probe would on its tick.
|
||||
func lookOnce(t *testing.T, j *Judge) {
|
||||
t.Helper()
|
||||
for _, d := range j.due() {
|
||||
ok, why := j.probes().Look(t.Context(), d.module, d.check)
|
||||
j.Record(d.id, d.started, ok, why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnHTTPCheckIsStartingUntilItPassesAndUnhealthyAfterItsLooksFail(t *testing.T) {
|
||||
answering := true
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !answering || r.URL.Path != "/healthz" {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.Write([]byte("ok"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := &clock{now: t0}
|
||||
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
||||
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
||||
web := Resource{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}
|
||||
j.Set([]Resource{web})
|
||||
|
||||
// In its grace, before any look: starting.
|
||||
st, _ := j.Look(t.Context())
|
||||
if s := stateOf(t, st, "app.web"); s.State != Starting || s.CheckOf() != "http" {
|
||||
t.Fatalf("before any look: %+v", s)
|
||||
}
|
||||
// It answers: healthy, even inside its grace.
|
||||
lookOnce(t, j)
|
||||
st, _ = j.Look(t.Context())
|
||||
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
||||
t.Fatalf("after a passing look: %+v", s)
|
||||
}
|
||||
// It stops answering, after its grace: one failing look is not yet unhealthy; the second is.
|
||||
answering = false
|
||||
c.now = t0.Add(time.Minute)
|
||||
lookOnce(t, j)
|
||||
st, _ = j.Look(t.Context())
|
||||
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
||||
t.Fatalf("one failing look made it %s", s.State)
|
||||
}
|
||||
c.now = c.now.Add(10 * time.Second)
|
||||
lookOnce(t, j)
|
||||
st, _ = j.Look(t.Context())
|
||||
s := stateOf(t, st, "app.web")
|
||||
if s.State != Unhealthy || !strings.Contains(s.Reason, "http /healthz on web: answered 503") {
|
||||
t.Fatalf("two failing looks: %+v", s)
|
||||
}
|
||||
// And nothing was restarted, recreated or stopped: the fake runtime was only read.
|
||||
answering = true
|
||||
c.now = c.now.Add(10 * time.Second)
|
||||
lookOnce(t, j)
|
||||
st, _ = j.Look(t.Context())
|
||||
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
||||
t.Fatalf("answering again: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnHTTPCheckDialsTheEndpointsCurrentPortAfterAPortChange(t *testing.T) {
|
||||
hit := map[string]int{}
|
||||
handler := func(name string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hit[name]++ })
|
||||
}
|
||||
before := httptest.NewServer(handler("before"))
|
||||
defer before.Close()
|
||||
after := httptest.NewServer(handler("after"))
|
||||
defer after.Close()
|
||||
c := &clock{now: t0}
|
||||
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
||||
j := aJudge(t, rt, c, "")
|
||||
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, before.URL))}})
|
||||
j.Look(t.Context())
|
||||
lookOnce(t, j)
|
||||
// The machine gave the endpoint another port: the next declaration says so, and the check follows.
|
||||
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, after.URL))}})
|
||||
j.Look(t.Context())
|
||||
lookOnce(t, j)
|
||||
if hit["before"] != 1 || hit["after"] != 1 {
|
||||
t.Fatalf("the check dialled %v; after the port moved it must dial the new one at once", hit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRuntimesCheckIsReadAndAnImageWithoutOneIsSaid(t *testing.T) {
|
||||
c := &clock{now: t0}
|
||||
o := running("c1", 0, t0.Format(time.RFC3339Nano))
|
||||
rt := &fakeRuntime{containers: map[string]Observed{"db": o}}
|
||||
j := aJudge(t, rt, c, "")
|
||||
check := &declaration.Health{Kind: declaration.HealthRuntime, Interval: "30s", Timeout: "5s", Looks: 3, Grace: "1m0s"}
|
||||
j.Set([]Resource{{Module: "app", ID: "app.db", Kind: KindContainer, Target: "db", Check: check}})
|
||||
look := func(health, said string) State {
|
||||
o.Health, o.HealthSaid = health, said
|
||||
rt.containers["db"] = o
|
||||
st, _ := j.Look(t.Context())
|
||||
return stateOf(t, st, "app.db")
|
||||
}
|
||||
if s := look("starting", ""); s.State != Starting {
|
||||
t.Fatalf("the runtime says starting: %+v", s)
|
||||
}
|
||||
if s := look(Healthy, ""); s.State != Healthy {
|
||||
t.Fatalf("the runtime says healthy: %+v", s)
|
||||
}
|
||||
c.now = t0.Add(2 * time.Minute)
|
||||
if s := look(Unhealthy, "curl: (7) Failed to connect to localhost port 3000"); s.State != Unhealthy ||
|
||||
!strings.Contains(s.Reason, "Failed to connect to localhost") {
|
||||
t.Fatalf("the runtime says unhealthy: %+v", s)
|
||||
}
|
||||
if s := look("", ""); s.State != Unhealthy || !strings.Contains(s.Reason, "ships no check to adopt") {
|
||||
t.Fatalf("an image with no check adopted by name: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAUnitIsReadyWhenItsManagerSaysItIsActive(t *testing.T) {
|
||||
c := &clock{now: t0}
|
||||
rt := &fakeRuntime{units: map[string]Observed{"d.service": {Found: true, Identity: "i1", Running: true}}}
|
||||
j := aJudge(t, rt, c, "")
|
||||
check := &declaration.Health{Kind: declaration.HealthUnit, Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
|
||||
j.Set([]Resource{{Module: "app", ID: "app.d", Kind: KindService, Target: "d.service", Check: check}})
|
||||
st, _ := j.Look(t.Context())
|
||||
if s := stateOf(t, st, "app.d"); s.State != Healthy {
|
||||
t.Fatalf("an active unit with no grace: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeverMoreLooksThanTheBudget(t *testing.T) {
|
||||
j := aJudge(t, &fakeRuntime{}, &clock{now: t0}, "")
|
||||
var rs []Resource
|
||||
for i := 0; i < 30; i++ {
|
||||
rs = append(rs, Resource{Module: "app", ID: "app.c" + strconv.Itoa(i), Kind: KindContainer, Target: "c" + strconv.Itoa(i),
|
||||
Check: &declaration.Health{Kind: declaration.HealthTCP, Port: 1, Interval: "10s", Timeout: "1s", Looks: 2, Grace: "0s"}})
|
||||
}
|
||||
j.Set(rs)
|
||||
// 30 checks every 10 s are 180 looks a minute: spaced twice their interval to stay within 90.
|
||||
if got := j.Spacing(); got != 2 {
|
||||
t.Fatalf("spaced %v", got)
|
||||
}
|
||||
j.Set(rs[:9])
|
||||
if got := j.Spacing(); got != 1 {
|
||||
t.Fatalf("54 looks a minute spaced %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
|
||||
asked := ""
|
||||
p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) {
|
||||
asked = module + "." + tool
|
||||
return false, "the admin refused the minted secret", nil
|
||||
}}
|
||||
ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
|
||||
Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"})
|
||||
if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") {
|
||||
t.Fatalf("asked %q: %v %q", asked, ok, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The runtime's state of a container's own check is read out of its whole state: a container that carries
|
||||
// none has no Health in it, and that is not an error (a template naming it would refuse every container).
|
||||
func TestTheRuntimesCheckIsReadOutOfTheContainersState(t *testing.T) {
|
||||
status, said := runtimeHealth(`{"Status":"running","Health":{"Status":"unhealthy","FailingStreak":3,"Log":[` +
|
||||
`{"ExitCode":1,"Output":"curl: (7) Failed to connect\n"}]}}`)
|
||||
if status != Unhealthy || said != "curl: (7) Failed to connect" {
|
||||
t.Errorf("read %q %q", status, said)
|
||||
}
|
||||
if status, _ := runtimeHealth(`{"Status":"running","Running":true}`); status != "" {
|
||||
t.Errorf("a container without a check read as %q", status)
|
||||
}
|
||||
if !strings.Contains(inspectFormat, "{{json .State}}") || strings.Contains(inspectFormat, ".State.Health") {
|
||||
t.Errorf("the inspect names a key a container without a check does not have: %s", inspectFormat)
|
||||
}
|
||||
}
|
||||
|
||||
// The engine's own looks run on their own clock, beside the looks of liveness, and are folded in.
|
||||
func TestProbeLooksOnItsOwnAndTheNextLookFoldsItIn(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer srv.Close()
|
||||
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, time.Now().Format(time.RFC3339Nano))}}
|
||||
j, err := Open("", rt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}})
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
go j.Probe(ctx)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if st, _ := j.Look(t.Context()); stateOf(t, st, "app.web").State == Healthy {
|
||||
return
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the engine's own look was never folded in")
|
||||
}
|
||||
@@ -7,10 +7,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
@@ -89,3 +91,114 @@ func TestReplayCrashLoopIsSaidUnhealthy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SilentWebImage and SilentWebProgram are the silent web application: a web server whose application
|
||||
// never answers — it accepts every request and holds it, as the application waiting on a database it could
|
||||
// not reach did. The lab raises the same (mesh-lab replays/silentweb_test.go).
|
||||
const (
|
||||
SilentWebImage = "busybox:1.36"
|
||||
SilentWebProgram = "mkdir -p /www/cgi-bin && printf '#!/bin/sh\\nsleep 3600\\n' > /www/cgi-bin/app && " +
|
||||
"chmod +x /www/cgi-bin/app && exec httpd -f -p 8080 -h /www"
|
||||
)
|
||||
|
||||
// **R145, the engine's half — a web application that accepts TCP and answers nothing is said unhealthy by
|
||||
// its HTTP check within two looks** (novox/hq ADR 0240 Phase B, issue 145). The application's port was
|
||||
// open and its program ran while every request hung, for eleven hours. Liveness says it alive and a TCP
|
||||
// check says it reachable; only the HTTP check its module declares sees it.
|
||||
//
|
||||
// A container that accepts every connection on its port and never answers is raised here as the
|
||||
// node-engine raises a module's, its port published on the machine; the judge looks at it with an HTTP check
|
||||
// of two looks and a TCP check beside it. MESH_REPLAY_STATEMENT, when set, is where the statement is written
|
||||
// for the controller's half (mesh-controller TestReplaySilentWebAppIsRaisedWithinTwoLooks). The container is
|
||||
// removed after, whatever happened.
|
||||
func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
|
||||
if os.Getenv("MESH_REPLAY_RUNTIME") != "1" && os.Getenv("MESH_REPLAY_CONTAINER") == "" {
|
||||
t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container")
|
||||
}
|
||||
run := func(ctx context.Context, cmd string, args ...string) (string, error) {
|
||||
c := exec.CommandContext(ctx, cmd, args...)
|
||||
out, err := c.Output()
|
||||
var exit *exec.ExitError
|
||||
if errors.As(err, &exit) {
|
||||
return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
||||
}
|
||||
return string(out), err
|
||||
}
|
||||
if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
|
||||
t.Skipf("no container runtime answers here: %v", err)
|
||||
}
|
||||
// The container is raised by the lab (MESH_REPLAY_CONTAINER, reached at MESH_REPLAY_ADDRESS on its own
|
||||
// port), or here, published on this machine's loopback.
|
||||
name, host, published := os.Getenv("MESH_REPLAY_CONTAINER"), os.Getenv("MESH_REPLAY_ADDRESS"), 8080
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("mesh-replay-silent-web-%d", time.Now().UnixNano())
|
||||
if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped",
|
||||
"--publish", "127.0.0.1::8080", "--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", SilentWebImage,
|
||||
"sh", "-c", SilentWebProgram); err != nil {
|
||||
t.Fatalf("starting the silent web application: %v %s", err, out)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) })
|
||||
said, err := run(t.Context(), "docker", "port", name, "8080/tcp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, port, _ := strings.Cut(strings.TrimSpace(strings.Split(said, "\n")[0]), "127.0.0.1:")
|
||||
if published, err = strconv.Atoi(port); err != nil {
|
||||
t.Fatalf("the port it was published on: %q", said)
|
||||
}
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
|
||||
timing := func(h *declaration.Health) *declaration.Health {
|
||||
h.Interval, h.Timeout, h.Looks, h.Grace = "10s", "2s", 2, "0s"
|
||||
return h
|
||||
}
|
||||
j, err := Open("", &Exec{Run: run})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.Set([]Resource{
|
||||
{Module: "app", ID: "app.server", Kind: KindContainer, Target: name,
|
||||
Check: timing(&declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: published, Path: "/cgi-bin/app"})},
|
||||
})
|
||||
j.Probes = &Probes{Host: host}
|
||||
tcp := j.Probes
|
||||
began := time.Now()
|
||||
var st Statement
|
||||
looks := 0
|
||||
for time.Since(began) < 2*time.Minute {
|
||||
for _, d := range j.due() {
|
||||
ok, why := j.probes().Look(t.Context(), d.module, d.check)
|
||||
j.Record(d.id, d.started, ok, why)
|
||||
looks++
|
||||
}
|
||||
st, _ = j.Look(t.Context())
|
||||
if st.Resources[0].State == Unhealthy {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
s := st.Resources[0]
|
||||
if s.State != Unhealthy || looks > 2 {
|
||||
t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s)
|
||||
}
|
||||
// And a TCP check would have said it reachable: the port is open.
|
||||
if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
|
||||
t.Fatalf("a tcp check could not connect to the silent application: %s", why)
|
||||
}
|
||||
t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks,
|
||||
time.Since(began).Round(time.Second))
|
||||
|
||||
if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" {
|
||||
h := link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{
|
||||
Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason,
|
||||
Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts, Check: s.CheckOf(), Needs: s.NeedsOf()}}}
|
||||
raw, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(out, raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package liveness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
@@ -52,8 +53,11 @@ func (e *Exec) runtime(ctx context.Context) (string, error) {
|
||||
}
|
||||
|
||||
// inspectFormat is what one inspect says of each container: its name, id, status, the runtime's restart
|
||||
// count and when its current run started.
|
||||
const inspectFormat = "{{.Name}}\t{{.Id}}\t{{.State.Status}}\t{{.RestartCount}}\t{{.State.StartedAt}}"
|
||||
// count, when its current run started — and the state of the check the runtime runs as the container's
|
||||
// own, read out of the whole state as one line of JSON, for a check declared exec or runtime (Phase B).
|
||||
// The whole state and not its Health: a container that carries no check has no such key, and the runtime
|
||||
// refuses a template naming a key that is not there — for every container in the read.
|
||||
const inspectFormat = "{{.Name}}\t{{.Id}}\t{{.State.Status}}\t{{.RestartCount}}\t{{.State.StartedAt}}\t{{json .State}}"
|
||||
|
||||
// Containers is one inspect of every container named. A name the runtime does not have is not found;
|
||||
// a runtime that does not answer is an error — unknown, never down.
|
||||
@@ -83,12 +87,47 @@ func (e *Exec) Containers(ctx context.Context, names []string) (map[string]Obser
|
||||
name := strings.TrimPrefix(strings.TrimSpace(parts[0]), "/")
|
||||
restarts, _ := strconv.ParseInt(strings.TrimSpace(parts[3]), 10, 64)
|
||||
status := strings.TrimSpace(parts[2])
|
||||
out[name] = Observed{Found: true, Identity: strings.TrimSpace(parts[1]), Running: status == "running",
|
||||
o := Observed{Found: true, Identity: strings.TrimSpace(parts[1]), Running: status == "running",
|
||||
Restarting: status == "restarting", Restarts: restarts, Started: strings.TrimSpace(parts[4])}
|
||||
if len(parts) > 5 {
|
||||
o.Health, o.HealthSaid = runtimeHealth(strings.Join(parts[5:], "\t"))
|
||||
}
|
||||
out[name] = o
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runtimeHealth reads, from a container's state, the runtime's state of its own check: its status and what its last look
|
||||
// printed, in a line. Nothing when the container carries no check.
|
||||
func runtimeHealth(said string) (string, string) {
|
||||
var state struct {
|
||||
Health *struct {
|
||||
Status string
|
||||
Log []struct {
|
||||
ExitCode int
|
||||
Output string
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(said)), &state); err != nil || state.Health == nil ||
|
||||
state.Health.Status == "" {
|
||||
return "", ""
|
||||
}
|
||||
h := state.Health
|
||||
last := ""
|
||||
if n := len(h.Log); n > 0 {
|
||||
l := h.Log[n-1]
|
||||
last = firstLine(l.Output)
|
||||
if last == "" && l.ExitCode != 0 {
|
||||
last = fmt.Sprintf("its check exited %d", l.ExitCode)
|
||||
}
|
||||
if len(last) > 200 {
|
||||
last = last[:200] + "…"
|
||||
}
|
||||
}
|
||||
return strings.ToLower(h.Status), last
|
||||
}
|
||||
|
||||
// missing is an inspect that failed only because a name is not there: what it printed for the others is
|
||||
// still the answer.
|
||||
func missing(err error) bool {
|
||||
|
||||
Reference in New Issue
Block a user