Write the self-check's findings and checked in mesh-sdk's health layout, so the condition keeper reads them as it reads every observer's (issue 1952)
Pins the group's mesh-sdk head 69fcb6c. A seat verb's run="true" is taken as the seat schema gives it.
This commit is contained in:
+63
-70
@@ -1,95 +1,88 @@
|
||||
package main
|
||||
|
||||
// A finding: what an observer writes for the condition keeper (novox/hq ADR 0564 §2). **A copy of the contract
|
||||
// mesh-sdk holds (issue 1952, S3)**: its layout is the controller's observation (mesh-controller
|
||||
// internal/conditions/condition.go), its key the controller's key rule, so a finding written here and the condition
|
||||
// the controller raised for the same fault share one key. When mesh-sdk's go/health carries the finding, this file
|
||||
// is replaced by it and nothing else changes.
|
||||
// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an
|
||||
// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong
|
||||
// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key
|
||||
// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The two severities a condition has (to-be 45 §2).
|
||||
// The severities and scopes, as mesh-sdk's health names them.
|
||||
const (
|
||||
Urgent = "urgent"
|
||||
Warning = "warning"
|
||||
Urgent = health.Urgent
|
||||
Warning = health.Warning
|
||||
ScopeBus = health.ScopeBus
|
||||
ScopeCore = health.ScopeCore
|
||||
ScopeProbe = health.ScopeProbe
|
||||
ScopeMesh = health.ScopeMesh
|
||||
)
|
||||
|
||||
// The scopes a key starts with, as the controller names them.
|
||||
const (
|
||||
ScopeBus = "bus"
|
||||
ScopeCore = "core"
|
||||
ScopeProbe = "probe"
|
||||
ScopeMesh = "mesh"
|
||||
)
|
||||
// Subject is what a finding is about.
|
||||
type Subject = health.Subject
|
||||
|
||||
// Subject is what a finding is about: its scope, its id within the scope, the machine it concerns.
|
||||
type Subject struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Also []string `json:"also,omitempty"`
|
||||
// Observation is one thing a probe sees wrong now.
|
||||
type Observation struct {
|
||||
Kind string
|
||||
Token string
|
||||
Subject Subject
|
||||
Severity health.Severity
|
||||
Summary string
|
||||
Said string
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
Needs string
|
||||
// Source is the probe that saw it, or the self-check itself for a probe that could not run.
|
||||
Source string
|
||||
// Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277).
|
||||
Confirm bool
|
||||
}
|
||||
|
||||
// Finding is one thing an observer sees wrong now, kept under its key in the observer's bucket `findings` and
|
||||
// deleted when it is no longer seen. Token and Confirm are the observer's own and never written.
|
||||
type Finding struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity string `json:"severity"`
|
||||
// Summary is one line in the mesh's words; Said the evidence (an address or a name stays here, never in the
|
||||
// summary, ADR 0234 §6).
|
||||
Summary string `json:"summary"`
|
||||
Said string `json:"said,omitempty"`
|
||||
// Headline, Explanation, Resolved and Needs are what the operator reads (ADR 0253).
|
||||
Headline string `json:"headline"`
|
||||
Explanation string `json:"explanation"`
|
||||
Resolved string `json:"resolved"`
|
||||
Needs string `json:"needs,omitempty"`
|
||||
// Source is what found it: a probe's id, or the observer itself for its own blindness.
|
||||
Source string `json:"source"`
|
||||
// Observer is the module that wrote it.
|
||||
Observer string `json:"observer"`
|
||||
|
||||
Token string `json:"-"`
|
||||
Confirm bool `json:"-"`
|
||||
}
|
||||
|
||||
// Key is where the finding is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
|
||||
func (f Finding) Key() string {
|
||||
token := f.Token
|
||||
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = f.Kind
|
||||
token = o.Kind
|
||||
}
|
||||
return Key(f.Subject.Scope, f.Subject.ID, token)
|
||||
return health.Key(o.Subject.Scope, o.Subject.ID, token)
|
||||
}
|
||||
|
||||
// Says is what the finding says, as written: two findings that say the same are one write (ADR 0289 §1).
|
||||
func (f Finding) Says() string {
|
||||
// Finding is the observation as written by run at, its evidence the one line it said.
|
||||
func (o Observation) Finding(run string, at time.Time) health.Finding {
|
||||
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
|
||||
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
|
||||
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}}
|
||||
if o.Token != "" && o.Token != o.Kind {
|
||||
f.Token = o.Token
|
||||
}
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said})
|
||||
return f
|
||||
}
|
||||
|
||||
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
|
||||
// (ADR 0289 §1).
|
||||
func says(f health.Finding) string {
|
||||
f.At, f.Run, f.V = time.Time{}, "", 0
|
||||
ev := make([]health.Evidence, len(f.Evidence))
|
||||
for i, e := range f.Evidence {
|
||||
ev[i] = health.Evidence{Said: e.Said}
|
||||
}
|
||||
f.Evidence = ev
|
||||
raw, _ := json.Marshal(f)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||
|
||||
// Key composes a key from its parts, each token made safe for the bus, as the controller's conditions.Key does.
|
||||
func Key(scope, id, token string) string {
|
||||
var parts []string
|
||||
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||
if p == "" {
|
||||
p = "_"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
// worded gives a finding the operator's words registered for its kind, where it says none of its own.
|
||||
func worded(f Finding) Finding {
|
||||
func worded(f Observation) Observation {
|
||||
w, ok := words[f.Kind]
|
||||
if !ok {
|
||||
return f
|
||||
@@ -146,8 +146,8 @@ func expectedConsumers() map[string][]buscompose.Consumer {
|
||||
}
|
||||
|
||||
// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head.
|
||||
func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) {
|
||||
var out []Finding
|
||||
func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
var out []Observation
|
||||
byStream := expectedConsumers()
|
||||
streams := make([]string, 0, len(byStream))
|
||||
for s := range byStream {
|
||||
@@ -175,21 +175,21 @@ func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) {
|
||||
who := want.Stream + "." + want.Name
|
||||
have, ok := held[want.Name]
|
||||
if !ok {
|
||||
out = append(out, Finding{Kind: "consumer-lost", Token: "missing", Severity: Warning,
|
||||
out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)),
|
||||
Said: "no consumer " + want.Name + " on " + want.Stream})
|
||||
continue
|
||||
}
|
||||
if differs := consumerDiffers(want, have); differs != "" {
|
||||
out = append(out, Finding{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs),
|
||||
Said: differs})
|
||||
}
|
||||
if want.Stream != "NODES" && have.Pending > behindBound {
|
||||
// One look at a consumer a burst put behind is not one that stays behind: held for the next run,
|
||||
// so healer H4 resets nothing on a single sample (novox/hq issue 277).
|
||||
out = append(out, Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending),
|
||||
Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)})
|
||||
@@ -220,7 +220,7 @@ func consumerDiffers(want buscompose.Consumer, have consumerRow) string {
|
||||
}
|
||||
|
||||
// probeStreams is D7: every stream the mesh defines exists as defined.
|
||||
func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) {
|
||||
func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
rows, err := c.streams("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -229,17 +229,17 @@ func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) {
|
||||
for _, r := range rows {
|
||||
held[r.Name] = r
|
||||
}
|
||||
var out []Finding
|
||||
var out []Observation
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
have, ok := held[s.Name]
|
||||
if !ok {
|
||||
out = append(out, Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||
Said: "no stream " + s.Name})
|
||||
continue
|
||||
}
|
||||
if differs := streamDiffers(s, have); differs != "" {
|
||||
out = append(out, Finding{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs),
|
||||
Said: differs})
|
||||
}
|
||||
@@ -293,7 +293,7 @@ type closedAnswer struct {
|
||||
}
|
||||
|
||||
// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour.
|
||||
func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) {
|
||||
func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err)
|
||||
@@ -306,7 +306,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) {
|
||||
if hours <= 0 {
|
||||
hours = 1
|
||||
}
|
||||
var out []Finding
|
||||
var out []Observation
|
||||
for _, u := range read.Users {
|
||||
if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax {
|
||||
continue // refused before it logged in is nobody's reconnect loop
|
||||
@@ -334,7 +334,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) {
|
||||
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
|
||||
}
|
||||
who, machine := busUserWords(u.User)
|
||||
out = append(out, Finding{Kind: "bus-reconnects", Severity: Warning,
|
||||
out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine},
|
||||
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+
|
||||
"reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax),
|
||||
@@ -370,9 +370,9 @@ func busUserWords(user string) (string, string) {
|
||||
|
||||
// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the
|
||||
// watchman's pulse on `checked`, is the other half.
|
||||
func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) {
|
||||
func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil {
|
||||
return []Finding{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"},
|
||||
return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"},
|
||||
Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " +
|
||||
"late signal is being said",
|
||||
Said: firstLine(err.Error())}}, nil
|
||||
@@ -383,7 +383,7 @@ func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) {
|
||||
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it
|
||||
// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip,
|
||||
// and no answer is no pass.
|
||||
func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) {
|
||||
func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
rows, err := c.streams("")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err)
|
||||
@@ -430,12 +430,12 @@ func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) {
|
||||
if len(problems) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []Finding{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"},
|
||||
return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"},
|
||||
Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil
|
||||
}
|
||||
|
||||
// sorted is findings in a stated order, so two runs over one mesh say the same.
|
||||
func sorted(out []Finding) []Finding {
|
||||
func sorted(out []Observation) []Observation {
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
|
||||
func keysOf(found []Finding) []string {
|
||||
func keysOf(found []Observation) []string {
|
||||
var out []string
|
||||
for _, f := range found {
|
||||
out = append(out, f.Key())
|
||||
@@ -100,7 +100,7 @@ func keysOf(found []Finding) []string {
|
||||
func TestTheBusProbes(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
probe func(context.Context, *Checker) ([]Finding, error)
|
||||
probe func(context.Context, *Checker) ([]Observation, error)
|
||||
bus bus
|
||||
want []string
|
||||
err string
|
||||
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Module and Seat are this module and the seat it holds once for the mesh.
|
||||
@@ -25,11 +27,11 @@ const (
|
||||
|
||||
// The states it keeps, and the event that ends every run.
|
||||
const (
|
||||
FindingsState = "findings"
|
||||
FindingsState = health.FindingsState
|
||||
VerdictState = "verdict"
|
||||
VerdictsState = "verdicts"
|
||||
LastKey = "last"
|
||||
CheckedEvent = "checked"
|
||||
CheckedEvent = health.EventChecked
|
||||
)
|
||||
|
||||
// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1).
|
||||
@@ -39,12 +41,12 @@ var (
|
||||
Within = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have.
|
||||
// The verdicts a probe can have, as mesh-sdk's health names them.
|
||||
const (
|
||||
VerdictPass = "pass"
|
||||
VerdictFail = "fail"
|
||||
VerdictFailedToRun = "failed-to-run"
|
||||
VerdictDeferred = "deferred"
|
||||
VerdictPass = health.VerdictPassed
|
||||
VerdictFail = health.VerdictFailed
|
||||
VerdictFailedToRun = health.VerdictFailedToRun
|
||||
VerdictDeferred = health.VerdictDeferred
|
||||
)
|
||||
|
||||
// sourceSelf is what writes a probe's failure to run: the self-check itself.
|
||||
@@ -68,43 +70,16 @@ type Probe struct {
|
||||
// Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6):
|
||||
// answered deferred, never a pass.
|
||||
Deferred string
|
||||
Run func(ctx context.Context, c *Checker) ([]Finding, error)
|
||||
Run func(ctx context.Context, c *Checker) ([]Observation, error)
|
||||
}
|
||||
|
||||
// ProbeVerdict is one probe's outcome in a run, as the controller's doctor says it.
|
||||
type ProbeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
FoundCount int `json:"found-count,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
// Counts are a run's verdicts, counted.
|
||||
type Counts struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
FailedToRun int `json:"failed-to-run"`
|
||||
Deferred int `json:"deferred"`
|
||||
}
|
||||
|
||||
// Run is one run of the registry, and the body of `checked`: the controller's doctor-heartbeat body unchanged
|
||||
// (ADR 0564 §6), Controller naming the machine that ran it.
|
||||
type Run struct {
|
||||
Run string `json:"run"`
|
||||
At time.Time `json:"at"`
|
||||
Started time.Time `json:"started"`
|
||||
Took string `json:"took"`
|
||||
IntervalSeconds int `json:"interval-seconds"`
|
||||
Counts Counts `json:"counts"`
|
||||
Probes []ProbeVerdict `json:"probes"`
|
||||
Controller string `json:"controller"`
|
||||
Why string `json:"why"`
|
||||
Unsaid int `json:"unsaid,omitempty"`
|
||||
}
|
||||
// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's
|
||||
// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it.
|
||||
type (
|
||||
ProbeVerdict = health.ProbeVerdict
|
||||
Counts = health.CheckedCounts
|
||||
Run = health.Checked
|
||||
)
|
||||
|
||||
// Checker is the self-check: the registry and what its probes reach.
|
||||
type Checker struct {
|
||||
@@ -135,7 +110,7 @@ func (c *Checker) LastRun() *Run {
|
||||
}
|
||||
|
||||
type result struct {
|
||||
found []Finding
|
||||
found []Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
@@ -169,9 +144,9 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
|
||||
written, readErr := c.written()
|
||||
seen := map[string]bool{}
|
||||
want := map[string]Finding{} // every finding this run writes, by key
|
||||
ran := map[string]bool{} // the sources whose findings this run judged whole
|
||||
var blind []Finding
|
||||
want := map[string]Observation{} // every finding this run writes, by key
|
||||
ran := map[string]bool{} // the sources whose findings this run judged whole
|
||||
var blind []Observation
|
||||
for i, p := range c.Registry {
|
||||
v := ProbeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
@@ -183,7 +158,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = VerdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
blind = append(blind, Finding{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true,
|
||||
blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
@@ -195,7 +170,6 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
}
|
||||
f.Source = p.ID
|
||||
f = worded(f)
|
||||
f.Observer = Module
|
||||
key := f.Key()
|
||||
seen[key] = true
|
||||
if f.Confirm && !c.seenBefore(key) && !written.has(key) {
|
||||
@@ -221,7 +195,6 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
ran[sourceSelf] = true
|
||||
for _, f := range blind {
|
||||
f = worded(f)
|
||||
f.Observer = Module
|
||||
key := f.Key()
|
||||
seen[key] = true
|
||||
if c.seenBefore(key) || written.has(key) {
|
||||
@@ -231,7 +204,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
if readErr != nil {
|
||||
// Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so.
|
||||
c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr)
|
||||
} else if err := c.write(written, want, ran); err != nil {
|
||||
} else if err := c.write(written, want, ran, run.Run); err != nil {
|
||||
c.Logf("[%s] what this run found is not all written: %v", Module, err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
@@ -277,8 +250,8 @@ func (c *Checker) seenBefore(key string) bool {
|
||||
return c.seen[key]
|
||||
}
|
||||
|
||||
// writtenFindings are the findings in the bucket, by key, with the source that wrote each and what it says.
|
||||
type writtenFindings map[string]Finding
|
||||
// writtenFindings are the findings in the bucket, by key.
|
||||
type writtenFindings map[string]health.Finding
|
||||
|
||||
func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok }
|
||||
|
||||
@@ -297,8 +270,8 @@ func (c *Checker) written() (writtenFindings, error) {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var f Finding
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the finding %s is not readable: %w", k, err)
|
||||
}
|
||||
out[k] = f
|
||||
@@ -308,19 +281,24 @@ func (c *Checker) written() (writtenFindings, error) {
|
||||
|
||||
// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding
|
||||
// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4).
|
||||
func (c *Checker) write(written writtenFindings, want map[string]Finding, ran map[string]bool) error {
|
||||
func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error {
|
||||
var errs []string
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
now := c.Now()
|
||||
for _, k := range keys {
|
||||
f := want[k]
|
||||
if was, ok := written[k]; ok && was.Says() == f.Says() {
|
||||
f := want[k].Finding(run, now)
|
||||
if was, ok := written[k]; ok && says(was) == says(f) {
|
||||
continue
|
||||
}
|
||||
if err := c.Store.Put(FindingsState, k, f); err != nil {
|
||||
raw, err := health.EncodeFinding(f)
|
||||
if err == nil {
|
||||
err = c.Store.Put(FindingsState, k, json.RawMessage(raw))
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, k+": "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// memStore is the bus's buckets in memory, counting the writes.
|
||||
@@ -91,29 +93,29 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) {
|
||||
// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can
|
||||
// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1).
|
||||
func TestARunWritesFindingsAsTheyChange(t *testing.T) {
|
||||
var found []Finding
|
||||
var found []Observation
|
||||
var fail error
|
||||
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Finding, error) { return found, fail }}
|
||||
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }}
|
||||
deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"}
|
||||
missing := Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"},
|
||||
missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"},
|
||||
Summary: "the stream EVENTS is not on the bus"}
|
||||
behind := Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"}
|
||||
store := newStore()
|
||||
c, said := checker(store, steady, deferred)
|
||||
|
||||
steps := []struct {
|
||||
name string
|
||||
found []Finding
|
||||
found []Observation
|
||||
fail error
|
||||
want []string // finding keys written after the run
|
||||
verdict string
|
||||
missingPs int // puts of the missing finding so far
|
||||
}{
|
||||
{"a finding is written", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"the same again is no write", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"one look can be wrong: held", []Finding{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"seen twice: written", []Finding{missing, behind}, nil,
|
||||
{"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"seen twice: written", []Observation{missing, behind}, nil,
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"blind once: clears nothing, held", nil, errors.New("timed out"),
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1},
|
||||
@@ -162,12 +164,12 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) {
|
||||
t.Errorf("the last verdict counts %+v", r.Counts)
|
||||
}
|
||||
// A finding carries the operator's words of its kind, the probe that found it and who observed it.
|
||||
found, fail = []Finding{missing}, nil
|
||||
found, fail = []Observation{missing}, nil
|
||||
c.RunOnce(context.Background(), "the test")
|
||||
raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing")
|
||||
var f Finding
|
||||
_ = json.Unmarshal(raw, &f)
|
||||
if f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || f.Explanation == "" {
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" ||
|
||||
f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" {
|
||||
t.Errorf("the written finding is %+v", f)
|
||||
}
|
||||
// Blind to its own bucket, a run writes and clears nothing.
|
||||
@@ -185,9 +187,9 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
|
||||
within := Within
|
||||
Within = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Within = within })
|
||||
for name, run := range map[string]func(context.Context, *Checker) ([]Finding, error){
|
||||
"panic": func(context.Context, *Checker) ([]Finding, error) { panic("boom") },
|
||||
"overrun": func(ctx context.Context, _ *Checker) ([]Finding, error) { time.Sleep(time.Second); return nil, nil },
|
||||
for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){
|
||||
"panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") },
|
||||
"overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil },
|
||||
} {
|
||||
c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run})
|
||||
c.Now = time.Now
|
||||
@@ -202,12 +204,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
|
||||
// every state the module keeps (ADR 0564 §12).
|
||||
func TestTheVerbsAndReads(t *testing.T) {
|
||||
store := newStore()
|
||||
many := make([]Finding, 25)
|
||||
many := make([]Observation, 25)
|
||||
for i := range many {
|
||||
many[i] = Finding{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"}
|
||||
many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"}
|
||||
}
|
||||
c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong",
|
||||
Run: func(context.Context, *Checker) ([]Finding, error) { return many, nil }})
|
||||
Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }})
|
||||
if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") {
|
||||
t.Fatalf("before a run: %v", err)
|
||||
}
|
||||
|
||||
@@ -197,9 +197,15 @@ func strArg(a map[string]any, k string) string {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// boolArg reads a flag as a boolean or as the seat's "true".
|
||||
func boolArg(a map[string]any, k string) bool {
|
||||
b, _ := a[k].(bool)
|
||||
return b
|
||||
switch v := a[k].(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
return strings.EqualFold(strings.TrimSpace(v), "true")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func intArg(a map[string]any, k string, otherwise int) int {
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
@@ -276,8 +277,8 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("the missing stream is not a finding on the bus: %v", err)
|
||||
}
|
||||
var f Finding
|
||||
if err := json.Unmarshal(e.Value(), &f); err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
|
||||
f, err := health.DecodeFinding(e.Value())
|
||||
if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
|
||||
t.Errorf("the finding is %s", e.Value())
|
||||
}
|
||||
verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-self-check
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
Reference in New Issue
Block a user