novox/mesh-host
6
0
Fork 0
Code Issues Pull Requests 2 Releases Wiki Activity

Merge pull request 'Node-engine: state a tool check that waits for the operator as waiting (issue 386, ADR 0283)' (#78) from feat/386-a-wait-for-the-operator into main

Browse Source
This commit was merged in pull request #78.
This commit is contained in:
mesh-admin
2026-10-10 20:03:27 +00:00
parent 9d8aa9d6f0 9545121ec9
commit 5226d7bcd0
13 changed files with 351 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/main.go
+2 -1
View File
@@ -1619,7 +1619,8 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health
for _, r := range st.Resources {
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User)})
Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User),
Waits: r.Waits})
}
if ns != nil && ns.State != "" {
h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}
go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
golang.org/x/sys v0.48.0
go.sum
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
internal/link/asktool_nats_test.go
+4 -4
View File
@@ -29,11 +29,11 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) {
q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node})
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
defer cancel()
healthy, why, err := q.AskTool(ctx, "sensors", "sensors_health")
if err != nil || !healthy {
t.Fatalf("a served healthy tool read as %v %q (%v)", healthy, why, err)
a, err := q.AskTool(ctx, "sensors", "sensors_health")
if err != nil || !a.Healthy {
t.Fatalf("a served healthy tool read as %+v (%v)", a, err)
}
_, _, err = q.AskTool(ctx, "sensors", "nothing_serves_this")
_, err = q.AskTool(ctx, "sensors", "nothing_serves_this")
if err == nil || !strings.Contains(err.Error(), "nothing on this machine answers") {
t.Fatalf("a tool nothing serves read as %v", err)
}
internal/link/health_test.go
+20 -5
View File
@@ -62,8 +62,8 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
`{"error":"503 no responders"}`: false,
`{"result":"yes"}`: false,
} {
got, _, err := ReadToolAnswer([]byte(reply))
if err != nil || got != want {
got, err := ReadToolAnswer([]byte(reply))
if err != nil || got.Healthy != want {
t.Errorf("%s read as %v (%v)", reply, got, err)
}
}
@@ -78,13 +78,13 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
// ask, the check fails and says which.
func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
q := &Queue{Membership: Membership{Node: "anchor"}}
if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
t.Fatalf("with no link open: %v", err)
}
quiet := newQuietLink()
q.attach(t.Context(), quiet)
if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
t.Fatalf("on a link that cannot ask: %v", err)
}
q.detach(quiet)
@@ -93,7 +93,7 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
// there), never stop at the queue's assertion.
held := &natsLink{node: "anchor"}
q.attach(t.Context(), held)
_, _, err := q.AskTool(t.Context(), "sensors", "sensors_health")
_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
if err == nil {
t.Fatal("asked on no connection and was answered")
}
@@ -101,3 +101,18 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
t.Fatalf("the NATS link was not asked: %v", err)
}
}
// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
// healthy, so nothing reading `healthy` alone takes it for a pass.
func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) {
got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
t.Fatalf("read as %+v (%v)", got, err)
}
h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
b, _ := json.Marshal(h)
if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
t.Fatalf("said on the wire as %s", b)
}
}
internal/link/messages.go
+9
View File
@@ -4,6 +4,8 @@ import (
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The wire formats shared with the control plane, which defines them separately because this
@@ -335,6 +337,10 @@ const (
StateHeld = "held"
// StateUnknown is a resource whose state could not be read.
StateUnknown = "unknown"
// StateWaiting is a resource whose tool check says it waits for the operator — a named own secret or
// setting of the module not given yet — and nothing else is wrong (novox/hq ADR 0283). Not a fault: the
// controller checks the wait, and an older controller reads it as not healthy and raises nothing.
StateWaiting = "waiting"
)
// The reasons an unhealthy resource is said with, when it is liveness that judged it.
@@ -376,6 +382,9 @@ type ResourceHealth struct {
// Search is how far the setuid search got, on an account's verdict that waits for it (novox/hq issue
// 361): Reason says the same in words. Absent otherwise.
Search *SetuidSearch `json:"search,omitempty"`
// Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one
// setting of the module by name, the part, and what the operator gives, in words. Absent otherwise.
Waits []health.Wait `json:"waits,omitempty"`
}
// SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times
internal/link/queue.go
+16 -15
View File
@@ -9,6 +9,8 @@ import (
"fmt"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1).
@@ -531,21 +533,21 @@ func (q *Queue) timeout() time.Duration {
// AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR
// 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open
// or nothing answered — said as the check's failure, never as healthy.
func (q *Queue) AskTool(ctx context.Context, module, tool string) (bool, string, error) {
func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return false, "", errors.New("no link to the bus is open")
return health.Answer{}, errors.New("no link to the bus is open")
}
asker, ok := bus.(ToolBus)
if !ok {
return false, "", errors.New("the link to the bus open now cannot ask a tool")
return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool")
}
reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}"))
if err != nil {
return false, "", err
return health.Answer{}, err
}
return ReadToolAnswer(reply)
}
@@ -572,24 +574,23 @@ func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error)
var ErrNotAsked = errors.New("not asked")
// ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call —
// as a health answer: an error is not healthy, and so is a result that does not say `healthy: true`.
func ReadToolAnswer(reply []byte) (bool, string, error) {
// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the
// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the
// SDK cannot check is unhealthy too, so a malformed one never excuses anything.
func ReadToolAnswer(reply []byte) (health.Answer, error) {
var env struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if err := json.Unmarshal(reply, &env); err != nil {
return false, "", fmt.Errorf("its answer is not one: %w", err)
return health.Answer{}, fmt.Errorf("its answer is not one: %w", err)
}
if env.Error != "" {
return false, env.Error, nil
return health.Unhealthy(env.Error), nil
}
var a struct {
Healthy bool `json:"healthy"`
Why string `json:"why"`
a, err := health.Read(env.Result)
if err != nil {
return health.Unhealthy("it answered something other than healthy or not"), nil
}
if err := json.Unmarshal(env.Result, &a); err != nil {
return false, "it answered something other than healthy or not", nil
}
return a.Healthy, a.Why, nil
return a, nil
}
internal/link/replay331_test.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
func TestReplay331(t *testing.T) {
q := &Queue{Membership: Membership{Node: "laptop"}}
q.attach(t.Context(), &natsLink{node: "laptop"})
_, _, err := q.AskTool(t.Context(), "sensors", "sensors_health")
_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
if err == nil {
t.Fatal("asked on no connection and was answered")
}
internal/liveness/liveness.go
+31 -4
View File
@@ -39,6 +39,7 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
"github.com/novox/mesh-host/internal/declaration"
)
@@ -66,6 +67,9 @@ const (
Starting = "starting"
Held = "held"
Unknown = "unknown"
// Waiting is a resource whose tool check says it waits for the operator, and nothing else is wrong
// (novox/hq ADR 0283): a statement, not a fault, said with what it waits for.
Waiting = "waiting"
ReasonDown = "down"
ReasonRestarting = "restarting"
@@ -175,6 +179,8 @@ type State struct {
Since time.Time `json:"since"`
Streak int `json:"streak,omitempty"`
Restarts int `json:"restarts,omitempty"`
// Waits is what a resource stated Waiting waits for (ADR 0283).
Waits []health.Wait `json:"waits,omitempty"`
}
// CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
@@ -235,6 +241,9 @@ type kept struct {
Passed bool `json:"passed,omitempty"`
Failing int `json:"failing,omitempty"`
Why string `json:"why,omitempty"`
// Waits is what the last look of a tool check said the resource waits for (ADR 0283), nil when it said
// healthy or unhealthy; Why then is the answer's own words.
Waits []health.Wait `json:"waits,omitempty"`
// probing says a look of the engine's own is under way; due when the next is.
probing bool
due time.Time
@@ -319,7 +328,7 @@ func (j *Judge) Set(resources []Resource) {
} else if ok && !sameCheck(k.Check, r.Check) {
// Its check changed — another kind, another port, another timing: what the old one found
// says nothing about the new, which looks again at once.
k.Check, k.Passed, k.Failing, k.Why, k.due = r.Check, false, 0, "", time.Time{}
k.Check, k.Passed, k.Failing, k.Why, k.Waits, k.due = r.Check, false, 0, "", nil, time.Time{}
}
}
for id := range j.f.Kept {
@@ -358,8 +367,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
seen[r.ID] = true
st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
Streak: k.Streak, Restarts: k.Counted})
word := k.State + "/" + k.Reason
Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)})
// What it waits for is part of what is said: a change of waits alone is a change.
word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k))
if j.said[r.ID] != word {
changed = true
j.said[r.ID] = word
@@ -392,7 +402,7 @@ func (j *Judge) judge(k *kept, o Observed, now time.Time, held, blind bool, why
k.Seen, k.Identity, k.RuntimeRestarts, k.RuntimeStarted = o.Found, o.Identity, o.Restarts, o.Started
k.Started, k.Recent = started, nil
// Every start is judged ready afresh (to-be 48 §4).
k.Passed, k.Failing, k.Why, k.due = false, 0, "", time.Time{}
k.Passed, k.Failing, k.Why, k.Waits, k.due = false, 0, "", nil, time.Time{}
}
// **Held is neither alive nor dead**, and the window ending is a start: judged from a fresh grace.
@@ -562,3 +572,20 @@ func firstLine(s string) string {
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
return line
}
// waitsOf is what a resource stated Waiting waits for, and nothing for any other state.
func waitsOf(k *kept) []health.Wait {
if k.State != Waiting {
return nil
}
return append([]health.Wait(nil), k.Waits...)
}
// waitsWord is a resource's waits in one comparable word.
func waitsWord(ws []health.Wait) string {
var b strings.Builder
for _, w := range ws {
b.WriteString(w.Part + "|" + w.Secret + "|" + w.Setting + "|" + w.What + ";")
}
return b.String()
}
internal/liveness/readiness.go
+33 -19
View File
@@ -14,6 +14,7 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
"github.com/novox/mesh-host/internal/declaration"
)
@@ -77,6 +78,10 @@ func ready(k *kept, o Observed, inGrace bool) (string, string) {
}
}
switch {
case len(k.Waits) > 0:
// **A wait for the operator is a statement, not a flake** (novox/hq ADR 0283): one look that says it
// is enough, inside the grace too; the next look that says anything else replaces it.
return Waiting, "waits for the operator: " + orSaid(k.Why, "it says what it waits for")
case !inGrace && k.Failing >= c.Looks:
return Unhealthy, c.Words() + ": " + orSaid(k.Why, "it failed")
case k.Passed:
@@ -156,8 +161,8 @@ func (j *Judge) Probe(ctx context.Context) {
}
go func(d dueLook) {
defer func() { <-slots }()
ok, why := j.probes().Look(ctx, d.module, d.check)
j.Record(d.id, d.started, ok, why)
ok, why, waits := j.probes().Look(ctx, d.module, d.check)
j.Record(d.id, d.started, ok, why, waits...)
}(d)
}
}
@@ -190,9 +195,10 @@ func (j *Judge) due() []dueLook {
}
// Record keeps what one of the engine's own looks found, for the next look to fold in: a pass is a pass
// whenever it came; a failure counts only after the grace. A look begun before the resource's current
// start says nothing about it.
func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
// whenever it came; a failure counts only after the grace; a look that says the resource waits for the
// operator (waits, ADR 0283) is kept as that, in the grace too, and counts as no failing look. A look begun
// before the resource's current start says nothing about it.
func (j *Judge) Record(id string, started time.Time, ok bool, why string, waits ...health.Wait) {
j.mu.Lock()
defer j.mu.Unlock()
k := j.f.Kept[id]
@@ -203,9 +209,13 @@ func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
if !k.Started.Equal(started) || k.Check == nil {
return
}
k.Waits = nil
switch {
case ok:
k.Passed, k.Failing, k.Why = true, 0, ""
case len(waits) > 0:
// A pass from before the wait says nothing now: a failing look after it counts from no pass.
k.Passed, k.Failing, k.Why, k.Waits = false, 0, why, append([]health.Wait(nil), waits...)
case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))):
k.Why = why
default:
@@ -227,11 +237,12 @@ type Probes struct {
// Host is where an endpoint's port is dialled: this machine, by default its loopback.
Host string
// AskTool asks a module's tool on this machine's own node tools; nil says it cannot be asked.
AskTool func(ctx context.Context, module, tool string) (healthy bool, why string, err error)
AskTool func(ctx context.Context, module, tool string) (health.Answer, error)
}
// Look makes one look of a check, and says whether it passed and, when not, what it found.
func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string) {
// Look makes one look of a check, and says whether it passed and, when not, what it found — and, for a
// tool that answers it waits for the operator (ADR 0283), what it waits for. Only a tool check can wait.
func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string, []health.Wait) {
timeout := c.Within()
if timeout <= 0 {
timeout = 5 * time.Second
@@ -240,30 +251,33 @@ func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health)
defer cancel()
switch c.Kind {
case declaration.HealthHTTP:
return p.http(ctx, c, timeout)
ok, why := p.http(ctx, c, timeout)
return ok, why, nil
case declaration.HealthTCP:
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", p.address(c.Port))
if err != nil {
return false, dialWords(err, timeout)
return false, dialWords(err, timeout), nil
}
_ = conn.Close()
return true, ""
return true, "", nil
case declaration.HealthTool:
if p.AskTool == nil {
return false, "its tool cannot be asked: this engine has no link to the node tools"
return false, "its tool cannot be asked: this engine has no link to the node tools", nil
}
healthy, why, err := p.AskTool(ctx, module, c.Tool)
a, err := p.AskTool(ctx, module, c.Tool)
switch {
case errors.Is(err, context.DeadlineExceeded):
return false, fmt.Sprintf("no answer within %s", timeout)
return false, fmt.Sprintf("no answer within %s", timeout), nil
case err != nil:
return false, firstLine(err.Error())
case !healthy:
return false, orSaid(why, "it answered not healthy, and not why")
return false, firstLine(err.Error()), nil
case a.State == health.StateWaits && len(a.Waits) > 0:
return false, orSaid(a.Why, "it waits for the operator, and not why"), a.Waits
case !a.Healthy:
return false, orSaid(a.Why, "it answered not healthy, and not why"), nil
}
return true, ""
return true, "", nil
}
return false, "the engine does not make a " + c.Kind + " look itself"
return false, "the engine does not make a " + c.Kind + " look itself", nil
}
func (p *Probes) address(port int) string {
internal/liveness/readiness_test.go
+12 -11
View File
@@ -13,6 +13,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
"github.com/novox/mesh-host/internal/declaration"
)
@@ -40,8 +41,8 @@ func portOf(t *testing.T, url string) int {
func lookOnce(t *testing.T, j *Judge) {
t.Helper()
for _, d := range j.due() {
ok, why := j.probes().Look(t.Context(), d.module, d.check)
j.Record(d.id, d.started, ok, why)
ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
j.Record(d.id, d.started, ok, why, waits...)
}
}
@@ -182,11 +183,11 @@ func TestNeverMoreLooksThanTheBudget(t *testing.T) {
func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
asked := ""
p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) {
p := &Probes{AskTool: func(_ context.Context, module, tool string) (health.Answer, error) {
asked = module + "." + tool
return false, "the admin refused the minted secret", nil
return health.Unhealthy("the admin refused the minted secret"), nil
}}
ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
ok, why, _ := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"})
if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") {
t.Fatalf("asked %q: %v %q", asked, ok, why)
@@ -198,16 +199,16 @@ func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
func TestAToolCheckThatGetsNoAnswerSaysHowLongItWaited(t *testing.T) {
check := &declaration.Health{Kind: declaration.HealthTool, Tool: "sensors_health",
Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
p := &Probes{AskTool: func(context.Context, string, string) (bool, string, error) {
return false, "", fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
p := &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
return health.Answer{}, fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
}}
if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
t.Fatalf("%v %q", ok, why)
}
p.AskTool = func(context.Context, string, string) (bool, string, error) {
return false, "", errors.New("no link to the bus is open")
p.AskTool = func(context.Context, string, string) (health.Answer, error) {
return health.Answer{}, errors.New("no link to the bus is open")
}
if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
t.Fatalf("%v %q", ok, why)
}
}
internal/liveness/replay_test.go
+3 -3
View File
@@ -168,8 +168,8 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
looks := 0
for time.Since(began) < 2*time.Minute {
for _, d := range j.due() {
ok, why := j.probes().Look(t.Context(), d.module, d.check)
j.Record(d.id, d.started, ok, why)
ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
j.Record(d.id, d.started, ok, why, waits...)
looks++
}
st, _ = j.Look(t.Context())
@@ -183,7 +183,7 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s)
}
// And a TCP check would have said it reachable: the port is open.
if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
if ok, why, _ := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
t.Fatalf("a tcp check could not connect to the silent application: %s", why)
}
t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks,
internal/liveness/waiting_test.go
+215
View File
@@ -0,0 +1,215 @@
package liveness
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
"github.com/novox/mesh-host/internal/declaration"
)
// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or
// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on
// one look, inside the grace too; the next look that says anything else replaces it.
var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"}
var toolCheck = &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s",
Looks: 2, Grace: "1m30s"}
// toolJudge is a judge of one running process whose tool check answers what *answer holds.
func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) {
j, c, _, _ := toolJudgeAt(t, answer, filepath.Join(t.TempDir(), FileName))
return j, c
}
// toolJudgeAt is toolJudge keeping its file at path, with its runtime.
func toolJudgeAt(t *testing.T, answer *[]byte, path string) (*Judge, *clock, *fakeRuntime, string) {
t.Helper()
c := &clock{now: t0}
rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}}
j := aJudge(t, rt, c, path)
j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
return health.Read(*answer)
}}
j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}})
j.Look(t.Context())
return j, c, rt, path
}
func answerOf(t *testing.T, a health.Answer) []byte {
t.Helper()
b, err := json.Marshal(a)
if err != nil {
t.Fatal(err)
}
return b
}
func next(t *testing.T, j *Judge, c *clock, by time.Duration) State {
t.Helper()
c.now = c.now.Add(by)
lookOnce(t, j)
st, _ := j.Look(t.Context())
return stateOf(t, st, "mounts.watch")
}
func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) {
answer := answerOf(t, health.Waiting("the source games waits for its password", password))
j, c := toolJudge(t, &answer)
s := next(t, j, c, 0)
if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" ||
!strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") {
t.Fatalf("in its grace, a tool that waits: %+v", s)
}
if s.Streak != 0 {
t.Fatalf("a wait counted as a failing streak: %+v", s)
}
}
func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) {
answer := answerOf(t, health.Waiting("waits", password))
j, c := toolJudge(t, &answer)
for i := 0; i < 5; i++ {
if s := next(t, j, c, 30*time.Second); s.State != Waiting {
t.Fatalf("look %d after the grace: %+v", i, s)
}
}
}
func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) {
answer := answerOf(t, health.Waiting("waits", password))
j, c := toolJudge(t, &answer)
next(t, j, c, 2*time.Minute)
answer = answerOf(t, health.Healthy("every source armed"))
if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 {
t.Fatalf("healthy after a wait: %+v", s)
}
}
func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) {
answer := answerOf(t, health.Waiting("waits", password))
j, c := toolJudge(t, &answer)
next(t, j, c, 2*time.Minute)
answer = answerOf(t, health.Unhealthy("the share is unreachable"))
s := next(t, j, c, 30*time.Second)
if s.State == Unhealthy || s.State == Waiting || s.State == Healthy || len(s.Waits) != 0 {
t.Fatalf("one failing look after a wait made it %+v", s)
}
if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") {
t.Fatalf("two failing looks after a wait: %+v", s)
}
}
func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) {
answer := []byte(`{"healthy":false,"why":"down"}`)
j, c := toolJudge(t, &answer)
next(t, j, c, 2*time.Minute)
if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 {
t.Fatalf("an old unhealthy answer: %+v", s)
}
answer = []byte(`{"healthy":true,"why":"ok"}`)
if s := next(t, j, c, 30*time.Second); s.State != Healthy {
t.Fatalf("an old healthy answer: %+v", s)
}
}
func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) {
answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`)
j, c := toolJudge(t, &answer)
next(t, j, c, 2*time.Minute)
s := next(t, j, c, 30*time.Second)
if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") {
t.Fatalf("a wait naming nothing: %+v", s)
}
}
// A pass from before a wait is not kept through it: passed, waited, then one failing look after the grace is
// not healthy.
func TestAPassBeforeAWaitIsNotKeptThroughIt(t *testing.T) {
answer := answerOf(t, health.Healthy("ok"))
j, c := toolJudge(t, &answer)
if s := next(t, j, c, 0); s.State != Healthy {
t.Fatalf("passing: %+v", s)
}
answer = answerOf(t, health.Waiting("waits", password))
next(t, j, c, 2*time.Minute)
answer = answerOf(t, health.Unhealthy("the share is unreachable"))
if s := next(t, j, c, 30*time.Second); s.State == Healthy || s.State == Waiting {
t.Fatalf("one failing look after a wait that followed a pass: %+v", s)
}
}
// A resource that is down is down, whatever its last look said it waits for.
func TestDownBeatsAWait(t *testing.T) {
answer := answerOf(t, health.Waiting("waits", password))
j, c, rt, _ := toolJudgeAt(t, &answer, filepath.Join(t.TempDir(), FileName))
next(t, j, c, 2*time.Minute)
rt.containers["watch"] = Observed{Found: true, Identity: "c1", Started: t0.Format(time.RFC3339Nano)}
c.now = c.now.Add(15 * time.Second)
st, _ := j.Look(t.Context())
if s := stateOf(t, st, "mounts.watch"); s.State != Unhealthy || s.Reason != ReasonDown || len(s.Waits) != 0 {
t.Fatalf("a stopped process that waited: %+v", s)
}
}
// A changed check says nothing of what the old one found waiting.
func TestAChangedCheckClearsTheWaits(t *testing.T) {
answer := answerOf(t, health.Waiting("waits", password))
j, c := toolJudge(t, &answer)
next(t, j, c, 2*time.Minute)
other := *toolCheck
other.Interval = "60s"
j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: &other}})
st, _ := j.Look(t.Context())
if s := stateOf(t, st, "mounts.watch"); s.State == Waiting || len(s.Waits) != 0 {
t.Fatalf("after its check changed: %+v", s)
}
}
// The engine restarted keeps what a resource waits for, as it keeps everything else it judged.
func TestWaitsAreKeptAcrossTheEnginesRestart(t *testing.T) {
answer := answerOf(t, health.Waiting("waits", password))
path := filepath.Join(t.TempDir(), FileName)
j, c, rt, _ := toolJudgeAt(t, &answer, path)
next(t, j, c, 2*time.Minute)
again := aJudge(t, rt, c, path)
again.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}})
st, _ := again.Look(t.Context())
if s := stateOf(t, st, "mounts.watch"); s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != password.Secret {
t.Fatalf("after the engine restarted: %+v", s)
}
}
// Waiting and healthy each way is a change, and so is a change of what it waits for alone.
func TestAChangeOfWaitingOrOfItsWaitsIsAChange(t *testing.T) {
answer := answerOf(t, health.Healthy("ok"))
j, c := toolJudge(t, &answer)
look := func(by time.Duration) bool {
c.now = c.now.Add(by)
lookOnce(t, j)
_, changed := j.Look(t.Context())
return changed
}
look(0)
answer = answerOf(t, health.Waiting("waits", password))
if !look(30 * time.Second) {
t.Fatal("healthy to waiting was no change")
}
if look(30 * time.Second) {
t.Fatal("the same wait again was a change")
}
user := health.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
answer = answerOf(t, health.Waiting("waits", password, user))
if !look(30 * time.Second) {
t.Fatal("a wait added was no change")
}
answer = answerOf(t, health.Healthy("ok"))
if !look(30 * time.Second) {
t.Fatal("waiting to healthy was no change")
}
}