From babe2f194a39d513fb64bc9d157468817f5caf6a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:48:09 +0100 Subject: [PATCH] Write the self-check's findings and checked in mesh-sdk's health layout, so the condition keeper reads them as it reads every observer's (issue 1952) Pins the group's mesh-sdk head 69fcb6c. A seat verb's run="true" is taken as the seat schema gives it. --- .../{finding.go => observation.go} | 133 +++++++++--------- .../cmd/mesh-self-check/probes.go | 34 ++--- .../cmd/mesh-self-check/probes_test.go | 4 +- .../cmd/mesh-self-check/selfcheck.go | 94 +++++-------- .../cmd/mesh-self-check/selfcheck_test.go | 40 +++--- .../cmd/mesh-self-check/tools.go | 10 +- .../mesh-self-check/whole_integration_test.go | 5 +- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +- 9 files changed, 153 insertions(+), 173 deletions(-) rename modules/mesh-self-check/cmd/mesh-self-check/{finding.go => observation.go} (50%) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/finding.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go similarity index 50% rename from modules/mesh-self-check/cmd/mesh-self-check/finding.go rename to modules/mesh-self-check/cmd/mesh-self-check/observation.go index 9a20352e..71209ce9 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/finding.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/observation.go @@ -1,95 +1,88 @@ package main -// A finding: what an observer writes for the condition keeper (novox/hq ADR 0564 §2). **A copy of the contract -// mesh-sdk holds (issue 1952, S3)**: its layout is the controller's observation (mesh-controller -// internal/conditions/condition.go), its key the controller's key rule, so a finding written here and the condition -// the controller raised for the same fault share one key. When mesh-sdk's go/health carries the finding, this file -// is replaced by it and nothing else changes. +// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an +// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong +// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key +// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. import ( "encoding/json" - "regexp" - "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) -// The two severities a condition has (to-be 45 §2). +// The severities and scopes, as mesh-sdk's health names them. const ( - Urgent = "urgent" - Warning = "warning" + Urgent = health.Urgent + Warning = health.Warning + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh ) -// The scopes a key starts with, as the controller names them. -const ( - ScopeBus = "bus" - ScopeCore = "core" - ScopeProbe = "probe" - ScopeMesh = "mesh" -) +// Subject is what a finding is about. +type Subject = health.Subject -// Subject is what a finding is about: its scope, its id within the scope, the machine it concerns. -type Subject struct { - Scope string `json:"scope"` - ID string `json:"id"` - Machine string `json:"machine,omitempty"` - Also []string `json:"also,omitempty"` +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is the probe that saw it, or the self-check itself for a probe that could not run. + Source string + // Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277). + Confirm bool } -// Finding is one thing an observer sees wrong now, kept under its key in the observer's bucket `findings` and -// deleted when it is no longer seen. Token and Confirm are the observer's own and never written. -type Finding struct { - Kind string `json:"kind"` - Subject Subject `json:"subject"` - Severity string `json:"severity"` - // Summary is one line in the mesh's words; Said the evidence (an address or a name stays here, never in the - // summary, ADR 0234 §6). - Summary string `json:"summary"` - Said string `json:"said,omitempty"` - // Headline, Explanation, Resolved and Needs are what the operator reads (ADR 0253). - Headline string `json:"headline"` - Explanation string `json:"explanation"` - Resolved string `json:"resolved"` - Needs string `json:"needs,omitempty"` - // Source is what found it: a probe's id, or the observer itself for its own blindness. - Source string `json:"source"` - // Observer is the module that wrote it. - Observer string `json:"observer"` - - Token string `json:"-"` - Confirm bool `json:"-"` -} - -// Key is where the finding is kept: `..`, the controller's key for the same condition. -func (f Finding) Key() string { - token := f.Token +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token if token == "" { - token = f.Kind + token = o.Kind } - return Key(f.Subject.Scope, f.Subject.ID, token) + return health.Key(o.Subject.Scope, o.Subject.ID, token) } -// Says is what the finding says, as written: two findings that say the same are one write (ADR 0289 §1). -func (f Finding) Says() string { +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev raw, _ := json.Marshal(f) return string(raw) } -var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) - -// Key composes a key from its parts, each token made safe for the bus, as the controller's conditions.Key does. -func Key(scope, id, token string) string { - var parts []string - for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { - p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") - if p == "" { - p = "_" - } - parts = append(parts, p) - } - return strings.Join(parts, ".") -} - // worded gives a finding the operator's words registered for its kind, where it says none of its own. -func worded(f Finding) Finding { +func worded(f Observation) Observation { w, ok := words[f.Kind] if !ok { return f diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 9f4e9882..44be64ce 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -146,8 +146,8 @@ func expectedConsumers() map[string][]buscompose.Consumer { } // probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. -func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) { - var out []Finding +func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) { + var out []Observation byStream := expectedConsumers() streams := make([]string, 0, len(byStream)) for s := range byStream { @@ -175,21 +175,21 @@ func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) { who := want.Stream + "." + want.Name have, ok := held[want.Name] if !ok { - out = append(out, Finding{Kind: "consumer-lost", Token: "missing", Severity: Warning, + out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), Said: "no consumer " + want.Name + " on " + want.Stream}) continue } if differs := consumerDiffers(want, have); differs != "" { - out = append(out, Finding{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, + out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), Said: differs}) } if want.Stream != "NODES" && have.Pending > behindBound { // One look at a consumer a burst put behind is not one that stays behind: held for the next run, // so healer H4 resets nothing on a single sample (novox/hq issue 277). - out = append(out, Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeBus, ID: who}, Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) @@ -220,7 +220,7 @@ func consumerDiffers(want buscompose.Consumer, have consumerRow) string { } // probeStreams is D7: every stream the mesh defines exists as defined. -func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) { +func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { rows, err := c.streams("") if err != nil { return nil, err @@ -229,17 +229,17 @@ func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) { for _, r := range rows { held[r.Name] = r } - var out []Finding + var out []Observation for _, s := range buscompose.MeshStreams() { have, ok := held[s.Name] if !ok { - out = append(out, Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), Said: "no stream " + s.Name}) continue } if differs := streamDiffers(s, have); differs != "" { - out = append(out, Finding{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), Said: differs}) } @@ -293,7 +293,7 @@ type closedAnswer struct { } // probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. -func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { +func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) if err != nil { return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) @@ -306,7 +306,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { if hours <= 0 { hours = 1 } - var out []Finding + var out []Observation for _, u := range read.Users { if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { continue // refused before it logged in is nobody's reconnect loop @@ -334,7 +334,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { partial = " (at least: the server's record of closed connections does not reach back the whole hour)" } who, machine := busUserWords(u.User) - out = append(out, Finding{Kind: "bus-reconnects", Severity: Warning, + out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), @@ -370,9 +370,9 @@ func busUserWords(user string) (string, string) { // probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the // watchman's pulse on `checked`, is the other half. -func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) { +func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { - return []Finding{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + "late signal is being said", Said: firstLine(err.Error())}}, nil @@ -383,7 +383,7 @@ func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) { // probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it // to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip, // and no answer is no pass. -func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) { +func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { rows, err := c.streams("") if err != nil { return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) @@ -430,12 +430,12 @@ func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) { if len(problems) == 0 { return nil, nil } - return []Finding{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil } // sorted is findings in a stated order, so two runs over one mesh say the same. -func sorted(out []Finding) []Finding { +func sorted(out []Observation) []Observation { sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) return out } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index 7b918757..825416e0 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -88,7 +88,7 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) { return nil, errors.New("nats: no responders available for request") } -func keysOf(found []Finding) []string { +func keysOf(found []Observation) []string { var out []string for _, f := range found { out = append(out, f.Key()) @@ -100,7 +100,7 @@ func keysOf(found []Finding) []string { func TestTheBusProbes(t *testing.T) { cases := []struct { name string - probe func(context.Context, *Checker) ([]Finding, error) + probe func(context.Context, *Checker) ([]Observation, error) bus bus want []string err string diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go index aa670108..8d07725d 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -15,6 +15,8 @@ import ( "strings" "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) // Module and Seat are this module and the seat it holds once for the mesh. @@ -25,11 +27,11 @@ const ( // The states it keeps, and the event that ends every run. const ( - FindingsState = "findings" + FindingsState = health.FindingsState VerdictState = "verdict" VerdictsState = "verdicts" LastKey = "last" - CheckedEvent = "checked" + CheckedEvent = health.EventChecked ) // The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). @@ -39,12 +41,12 @@ var ( Within = 30 * time.Second ) -// The verdicts a probe can have. +// The verdicts a probe can have, as mesh-sdk's health names them. const ( - VerdictPass = "pass" - VerdictFail = "fail" - VerdictFailedToRun = "failed-to-run" - VerdictDeferred = "deferred" + VerdictPass = health.VerdictPassed + VerdictFail = health.VerdictFailed + VerdictFailedToRun = health.VerdictFailedToRun + VerdictDeferred = health.VerdictDeferred ) // sourceSelf is what writes a probe's failure to run: the self-check itself. @@ -68,43 +70,16 @@ type Probe struct { // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): // answered deferred, never a pass. Deferred string - Run func(ctx context.Context, c *Checker) ([]Finding, error) + Run func(ctx context.Context, c *Checker) ([]Observation, error) } -// ProbeVerdict is one probe's outcome in a run, as the controller's doctor says it. -type ProbeVerdict struct { - ID string `json:"id"` - Verdict string `json:"verdict"` - Found []string `json:"found,omitempty"` - Unconfirmed []string `json:"unconfirmed,omitempty"` - Error string `json:"error,omitempty"` - Took string `json:"took,omitempty"` - FoundCount int `json:"found-count,omitempty"` - More string `json:"more,omitempty"` -} - -// Counts are a run's verdicts, counted. -type Counts struct { - Passed int `json:"passed"` - Failed int `json:"failed"` - FailedToRun int `json:"failed-to-run"` - Deferred int `json:"deferred"` -} - -// Run is one run of the registry, and the body of `checked`: the controller's doctor-heartbeat body unchanged -// (ADR 0564 §6), Controller naming the machine that ran it. -type Run struct { - Run string `json:"run"` - At time.Time `json:"at"` - Started time.Time `json:"started"` - Took string `json:"took"` - IntervalSeconds int `json:"interval-seconds"` - Counts Counts `json:"counts"` - Probes []ProbeVerdict `json:"probes"` - Controller string `json:"controller"` - Why string `json:"why"` - Unsaid int `json:"unsaid,omitempty"` -} +// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's +// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it. +type ( + ProbeVerdict = health.ProbeVerdict + Counts = health.CheckedCounts + Run = health.Checked +) // Checker is the self-check: the registry and what its probes reach. type Checker struct { @@ -135,7 +110,7 @@ func (c *Checker) LastRun() *Run { } type result struct { - found []Finding + found []Observation err error took time.Duration } @@ -169,9 +144,9 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { written, readErr := c.written() seen := map[string]bool{} - want := map[string]Finding{} // every finding this run writes, by key - ran := map[string]bool{} // the sources whose findings this run judged whole - var blind []Finding + want := map[string]Observation{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Observation for i, p := range c.Registry { v := ProbeVerdict{ID: p.ID} r := results[i] @@ -183,7 +158,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { case r.err != nil: v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() run.Counts.FailedToRun++ - blind = append(blind, Finding{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, + blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), Said: firstLine(r.err.Error())}) @@ -195,7 +170,6 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { } f.Source = p.ID f = worded(f) - f.Observer = Module key := f.Key() seen[key] = true if f.Confirm && !c.seenBefore(key) && !written.has(key) { @@ -221,7 +195,6 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { ran[sourceSelf] = true for _, f := range blind { f = worded(f) - f.Observer = Module key := f.Key() seen[key] = true if c.seenBefore(key) || written.has(key) { @@ -231,7 +204,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { if readErr != nil { // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) - } else if err := c.write(written, want, ran); err != nil { + } else if err := c.write(written, want, ran, run.Run); err != nil { c.Logf("[%s] what this run found is not all written: %v", Module, err) } c.mu.Lock() @@ -277,8 +250,8 @@ func (c *Checker) seenBefore(key string) bool { return c.seen[key] } -// writtenFindings are the findings in the bucket, by key, with the source that wrote each and what it says. -type writtenFindings map[string]Finding +// writtenFindings are the findings in the bucket, by key. +type writtenFindings map[string]health.Finding func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } @@ -297,8 +270,8 @@ func (c *Checker) written() (writtenFindings, error) { if !found { continue } - var f Finding - if err := json.Unmarshal(raw, &f); err != nil { + f, err := health.DecodeFinding(raw) + if err != nil { return out, fmt.Errorf("the finding %s is not readable: %w", k, err) } out[k] = f @@ -308,19 +281,24 @@ func (c *Checker) written() (writtenFindings, error) { // write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding // of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). -func (c *Checker) write(written writtenFindings, want map[string]Finding, ran map[string]bool) error { +func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error { var errs []string keys := make([]string, 0, len(want)) for k := range want { keys = append(keys, k) } sort.Strings(keys) + now := c.Now() for _, k := range keys { - f := want[k] - if was, ok := written[k]; ok && was.Says() == f.Says() { + f := want[k].Finding(run, now) + if was, ok := written[k]; ok && says(was) == says(f) { continue } - if err := c.Store.Put(FindingsState, k, f); err != nil { + raw, err := health.EncodeFinding(f) + if err == nil { + err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { errs = append(errs, k+": "+err.Error()) } } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index 42793250..311bf924 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -11,6 +11,8 @@ import ( "sync" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) // memStore is the bus's buckets in memory, counting the writes. @@ -91,29 +93,29 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { // could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can // be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). func TestARunWritesFindingsAsTheyChange(t *testing.T) { - var found []Finding + var found []Observation var fail error - steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Finding, error) { return found, fail }} + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }} deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} - missing := Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "the stream EVENTS is not on the bus"} - behind := Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} store := newStore() c, said := checker(store, steady, deferred) steps := []struct { name string - found []Finding + found []Observation fail error want []string // finding keys written after the run verdict string missingPs int // puts of the missing finding so far }{ - {"a finding is written", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"the same again is no write", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"one look can be wrong: held", []Finding{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"seen twice: written", []Finding{missing, behind}, nil, + {"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []Observation{missing, behind}, nil, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, {"blind once: clears nothing, held", nil, errors.New("timed out"), []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, @@ -162,12 +164,12 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { t.Errorf("the last verdict counts %+v", r.Counts) } // A finding carries the operator's words of its kind, the probe that found it and who observed it. - found, fail = []Finding{missing}, nil + found, fail = []Observation{missing}, nil c.RunOnce(context.Background(), "the test") raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") - var f Finding - _ = json.Unmarshal(raw, &f) - if f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || f.Explanation == "" { + f, err := health.DecodeFinding(raw) + if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || + f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { t.Errorf("the written finding is %+v", f) } // Blind to its own bucket, a run writes and clears nothing. @@ -185,9 +187,9 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { within := Within Within = 50 * time.Millisecond t.Cleanup(func() { Within = within }) - for name, run := range map[string]func(context.Context, *Checker) ([]Finding, error){ - "panic": func(context.Context, *Checker) ([]Finding, error) { panic("boom") }, - "overrun": func(ctx context.Context, _ *Checker) ([]Finding, error) { time.Sleep(time.Second); return nil, nil }, + for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){ + "panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil }, } { c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) c.Now = time.Now @@ -202,12 +204,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { // every state the module keeps (ADR 0564 §12). func TestTheVerbsAndReads(t *testing.T) { store := newStore() - many := make([]Finding, 25) + many := make([]Observation, 25) for i := range many { - many[i] = Finding{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} } c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", - Run: func(context.Context, *Checker) ([]Finding, error) { return many, nil }}) + Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }}) if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { t.Fatalf("before a run: %v", err) } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go index b38fa830..1c1df753 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/tools.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -197,9 +197,15 @@ func strArg(a map[string]any, k string) string { return strings.TrimSpace(s) } +// boolArg reads a flag as a boolean or as the seat's "true". func boolArg(a map[string]any, k string) bool { - b, _ := a[k].(bool) - return b + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false } func intArg(a map[string]any, k string, otherwise int) int { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go index 8b82b408..cc078779 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -26,6 +26,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/testbus" ) @@ -276,8 +277,8 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { if err != nil { t.Fatalf("the missing stream is not a finding on the bus: %v", err) } - var f Finding - if err := json.Unmarshal(e.Value(), &f); err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + f, err := health.DecodeFinding(e.Value()) + if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { t.Errorf("the finding is %s", e.Value()) } verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index abaa962a..a4d5f850 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 043e05f0..ae67790f 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=