From 2978dd5bedd05397dd05ff8c1d89fc0b4d2c2e31 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 17:05:27 +0100 Subject: [PATCH] =?UTF-8?q?mesh-healers=20and=20mesh-self-check=20use=20th?= =?UTF-8?q?e=20conditions=20client's=20Observation,=20Finding=20encoding?= =?UTF-8?q?=20and=20Says,=20not=20copies=20of=20their=20own=20(ADR=200634?= =?UTF-8?q?=20=C2=A74;=20issue=201966)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../mesh-healers/cmd/mesh-healers/healers.go | 30 ++-- .../cmd/mesh-healers/healers_test.go | 10 +- .../cmd/mesh-healers/observation.go | 78 ---------- .../mesh-healers/cmd/mesh-healers/wanted.go | 9 +- .../mesh-healers/whole_integration_test.go | 3 +- .../cmd/mesh-self-check/main.go | 5 +- .../cmd/mesh-self-check/observation.go | 133 ------------------ .../cmd/mesh-self-check/probes.go | 27 ++-- .../cmd/mesh-self-check/probes_data.go | 24 ++-- .../cmd/mesh-self-check/probes_fill.go | 17 +-- .../cmd/mesh-self-check/probes_mesh.go | 15 +- .../cmd/mesh-self-check/probes_root.go | 16 ++- .../cmd/mesh-self-check/probes_test.go | 18 +-- .../cmd/mesh-self-check/selfcheck.go | 23 +-- .../cmd/mesh-self-check/selfcheck_test.go | 47 ++++--- .../mesh-self-check/whole_integration_test.go | 3 +- .../cmd/mesh-self-check/words.go | 62 ++++++++ 17 files changed, 195 insertions(+), 325 deletions(-) delete mode 100644 modules/mesh-healers/cmd/mesh-healers/observation.go delete mode 100644 modules/mesh-self-check/cmd/mesh-self-check/observation.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/words.go diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index e7e335db1..92138a3dd 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -98,12 +98,12 @@ var ErrExists = errors.New("the key was made by another writer first") // Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside. type Condition struct { - Key string `json:"key"` - Kind string `json:"kind"` - Subject Subject `json:"subject"` - Severity string `json:"severity"` - Resolver string `json:"resolver"` - Tried []json.RawMessage `json:"tried,omitempty"` + Key string `json:"key"` + Kind string `json:"kind"` + Subject meshconditions.Subject `json:"subject"` + Severity string `json:"severity"` + Resolver string `json:"resolver"` + Tried []json.RawMessage `json:"tried,omitempty"` } // Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until @@ -240,7 +240,7 @@ func (h *Healers) Judge(c Condition, by string) (string, error) { h.see("blind", nil) acts := actsWithin(all, now.Add(-BrakeWindow)) if braked, said := h.brakeHolds(acts, now); braked { - h.see("brake", []Observation{brakeFinding(acts, said)}) + h.see("brake", []meshconditionsclient.Observation{brakeFinding(acts, said)}) return "every healer has stopped: " + said, nil } h.see("brake", nil) @@ -483,10 +483,10 @@ func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) { return false, said } -var brakeKey = meshconditionsclient.Key(ScopeMesh, "healers", "braked") +var brakeKey = meshconditionsclient.Key(meshconditionsclient.ScopeMesh, "healers", "braked") // brakeFinding is the brake, as the urgent finding that says it. -func brakeFinding(acts []Attempt, said string) Observation { +func brakeFinding(acts []Attempt, said string) meshconditionsclient.Observation { counts := map[string]int{} for _, a := range acts { counts[a.Healer+" on "+a.Condition]++ @@ -496,7 +496,7 @@ func brakeFinding(acts []Attempt, said string) Observation { most = append(most, fmt.Sprintf("%s ×%d", what, n)) } sort.Strings(most) - return Observation{Kind: "healers-braked", Ref: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"}, + return meshconditionsclient.Observation{Kind: "healers-braked", Ref: "braked", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeMesh, ID: "healers", Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " + "healer looping, not repairing — " + said, Said: strings.Join(most, ", "), @@ -509,8 +509,8 @@ func brakeFinding(acts []Attempt, said string) Observation { // blind writes the healers' own blindness: they cannot read what they did. func (h *Healers) blind(err error) { - h.see("blind", []Observation{{Kind: "probe-failed", Ref: "failed", Severity: Warning, - Subject: Subject{Scope: ScopeProbe, ID: "healers"}, + h.see("blind", []meshconditionsclient.Observation{{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning, + Scope: meshconditionsclient.ScopeProbe, ID: "healers", Summary: "the healers cannot read what they did, so they count no budget and act on nothing", Said: firstLine(err.Error()), Headline: "A check of the mesh could not run", Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.", @@ -519,12 +519,12 @@ func (h *Healers) blind(err error) { // see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it // no longer sees (ADR 0564 §2). -func (h *Healers) see(source string, found []Observation) { +func (h *Healers) see(source string, found []meshconditionsclient.Observation) { now := h.Now() want := map[string]meshconditions.FindingsByKeyValue{} for _, o := range found { o.Source = source - want[o.Key()] = o.Finding("", now) + want[o.Key()] = o.Finding(Module, "", now) } keys, err := h.Store.Keys(FindingsState) if err != nil { @@ -542,7 +542,7 @@ func (h *Healers) see(source string, found []Observation) { continue } if f, still := want[k]; still { - if says(was) == says(f) { + if meshconditionsclient.Says(was) == meshconditionsclient.Says(f) { delete(want, k) } continue diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index d7e5d68fd..5dff69eca 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "slices" "sort" "strings" @@ -12,6 +11,9 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts" ) @@ -171,7 +173,7 @@ var drill = Healer{ID: "HD", Kinds: []string{"drill-broken"}, Condition: "a dril // lost is a condition the drill healer answers, on a module. func lost(module string) Condition { return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "drill-broken", Severity: "warning", Resolver: "self", - Subject: Subject{Scope: "bus", ID: "EVENTS." + module}} + Subject: meshconditions.Subject{Scope: "bus", ID: "EVENTS." + module}} } func body(c Condition) []byte { @@ -288,7 +290,7 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { raw, braked, _ := b.store.Get(FindingsState, brakeKey) var f meshconditions.FindingsByKeyValue _ = json.Unmarshal(raw, &f) - if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module { + if !braked || f.Severity != meshconditionsclient.Urgent || f.Kind != "healers-braked" || f.Observer != Module { t.Errorf("the brake is written as %s", raw) } b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held @@ -319,7 +321,7 @@ func TestTheRegistryActsOnNothingHere(t *testing.T) { w := newWorld() w.h.Registry = Registry() for _, kind := range []string{"consumer-lost", "holder-silent", "consumer-behind"} { - c := Condition{Key: "bus.EVENTS.records." + kind, Kind: kind, Subject: Subject{Scope: "bus", ID: "EVENTS.records"}} + c := Condition{Key: "bus.EVENTS.records." + kind, Kind: kind, Subject: meshconditions.Subject{Scope: "bus", ID: "EVENTS.records"}} if said, _ := w.h.Judge(c, ""); len(w.asked) != 0 || !strings.Contains(said, "no healer here") { t.Errorf("%s: %q, asked %v", kind, said, w.asked) } diff --git a/modules/mesh-healers/cmd/mesh-healers/observation.go b/modules/mesh-healers/cmd/mesh-healers/observation.go deleted file mode 100644 index 6adccda58..000000000 --- a/modules/mesh-healers/cmd/mesh-healers/observation.go +++ /dev/null @@ -1,78 +0,0 @@ -package main - -// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their -// blindness and S15, each written into the bucket `findings` as the seat mesh-conditions' finding, whose key is the -// controller's key for the same condition. - -import ( - "encoding/json" - "time" - - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" -) - -// The severities and scopes, as mesh-sdk's health names them. -const ( - Urgent = meshconditionsclient.Urgent - Warning = meshconditionsclient.Warning - ScopeProbe = meshconditionsclient.ScopeProbe - ScopeMesh = meshconditionsclient.ScopeMesh -) - -// Subject is what a finding is about. -type Subject = meshconditions.Subject - -// Observation is one thing a probe sees wrong now. -type Observation struct { - Kind string - Ref string - Subject Subject - Severity string - Summary string - Said string - Headline string - Explanation string - Resolved string - Needs string - // Source is what saw it: the brake, the healers' blindness, or S15. - Source string -} - -// Key is where it is kept: `..`, the controller's key for the same condition. -func (o Observation) Key() string { - token := o.Ref - if token == "" { - token = o.Kind - } - return meshconditionsclient.Key(o.Subject.Scope, o.Subject.ID, token) -} - -// Finding is the observation as written by run at, its evidence the one line it said. -func (o Observation) Finding(run string, at time.Time) meshconditions.FindingsByKeyValue { - f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, - Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, - Observer: Module, Run: run, At: at.UTC(), Evidence: []meshconditions.Evidence{}} - if o.Ref != "" && o.Ref != o.Kind { - f.Ref = o.Ref - } - said := o.Said - if said == "" { - said = o.Summary - } - f.Evidence = append(f.Evidence, meshconditions.Evidence{At: at.UTC(), Said: said}) - return f -} - -// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write -// (ADR 0289 §1). -func says(f meshconditions.FindingsByKeyValue) string { - f.At, f.Run, f.V = time.Time{}, "", 0 - ev := make([]meshconditions.Evidence, len(f.Evidence)) - for i, e := range f.Evidence { - ev[i] = meshconditions.Evidence{Said: e.Said} - } - f.Evidence = ev - raw, _ := json.Marshal(f) - return string(raw) -} diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go index cbee1a0e3..f664d19ad 100644 --- a/modules/mesh-healers/cmd/mesh-healers/wanted.go +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts" ) @@ -107,7 +108,7 @@ func (h *Healers) Seeded(answer []byte) (int, error) { // Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, // deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. -func (h *Healers) Wanted() []Observation { +func (h *Healers) Wanted() []meshconditionsclient.Observation { h.mu.Lock() defer h.mu.Unlock() now := h.Now() @@ -125,7 +126,7 @@ func (h *Healers) Wanted() []Observation { } } sort.Strings(causes) - var found []Observation + var found []meshconditionsclient.Observation var next time.Time for _, cause := range causes { acts := byCause[cause] @@ -144,8 +145,8 @@ func (h *Healers) Wanted() []Observation { } } words := strings.NewReplacer(".", " ", "_", " ").Replace(cause) - found = append(found, Observation{Kind: "healer-wanted", Ref: "healer-wanted", Severity: Warning, - Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause}, + found = append(found, meshconditionsclient.Observation{Kind: "healer-wanted", Ref: "healer-wanted", Severity: meshconditionsclient.Warning, + Scope: meshconditionsclient.ScopeMesh, ID: "hand-acts." + cause, Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts), int(Within.Hours()/24), newest.By, wanted), Said: strings.Join(lines, "; "), diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go index 715337384..926a30d76 100644 --- a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -15,7 +15,6 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "io" "os" "os/exec" @@ -25,6 +24,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go index 6ddc930a6..119da72cd 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/main.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -9,11 +9,12 @@ import ( "context" "encoding/json" "fmt" - assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" - placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "os" "time" + assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" + placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" stdio "git.novox.be/novox/mesh-sdk/go" diff --git a/modules/mesh-self-check/cmd/mesh-self-check/observation.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go deleted file mode 100644 index de29cd4af..000000000 --- a/modules/mesh-self-check/cmd/mesh-self-check/observation.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an -// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong -// about it. It is written into the bucket `findings` as the seat mesh-conditions' finding, whose key is the controller's key -// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. - -import ( - "encoding/json" - "time" - - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" -) - -// The severities and scopes, as mesh-sdk's health names them. -const ( - Urgent = meshconditionsclient.Urgent - Warning = meshconditionsclient.Warning - ScopeBus = meshconditionsclient.ScopeBus - ScopeCore = meshconditionsclient.ScopeCore - ScopeProbe = meshconditionsclient.ScopeProbe - ScopeMesh = meshconditionsclient.ScopeMesh -) - -// Subject is what a finding is about. -type Subject = meshconditions.Subject - -// Observation is one thing a probe sees wrong now. -type Observation struct { - Kind string - Ref string - Subject Subject - Severity string - Summary string - Said string - Headline string - Explanation string - Resolved string - Needs string - // Source is the probe that saw it, or the self-check itself for a probe that could not run. - Source string - // Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277). - Confirm bool -} - -// Key is where it is kept: `..`, the controller's key for the same condition. -func (o Observation) Key() string { - token := o.Ref - if token == "" { - token = o.Kind - } - return meshconditionsclient.Key(o.Subject.Scope, o.Subject.ID, token) -} - -// Finding is the observation as written by run at, its evidence the one line it said. -func (o Observation) Finding(run string, at time.Time) meshconditions.FindingsByKeyValue { - f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, - Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, - Observer: Module, Run: run, At: at.UTC(), Evidence: []meshconditions.Evidence{}} - if o.Ref != "" && o.Ref != o.Kind { - f.Ref = o.Ref - } - said := o.Said - if said == "" { - said = o.Summary - } - f.Evidence = append(f.Evidence, meshconditions.Evidence{At: at.UTC(), Said: said}) - return f -} - -// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write -// (ADR 0289 §1). -func says(f meshconditions.FindingsByKeyValue) string { - f.At, f.Run, f.V = time.Time{}, "", 0 - ev := make([]meshconditions.Evidence, len(f.Evidence)) - for i, e := range f.Evidence { - ev[i] = meshconditions.Evidence{Said: e.Said} - } - f.Evidence = ev - raw, _ := json.Marshal(f) - return string(raw) -} - -// worded gives a finding the operator's words registered for its kind, where it says none of its own. -func worded(f Observation) Observation { - w, ok := words[f.Kind] - if !ok { - return f - } - if f.Headline == "" { - f.Headline = w.Headline - } - if f.Explanation == "" { - f.Explanation = w.Explanation - } - if f.Resolved == "" { - f.Resolved = w.Resolved - } - if f.Needs == "" { - f.Needs = w.Needs - } - return f -} - -type plain struct{ Headline, Explanation, Resolved, Needs string } - -// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the -// controller's name for itself replaced by the self-check's where the words spoke of it. -var words = map[string]plain{ - "stream-wrong": {Headline: "Part of the bus's storage is wrong", - Needs: "check the machine the bus runs on; the details say what is missing.", - Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", - Resolved: "The bus's storage is right again"}, - "bus-reconnects": {Headline: "A client keeps losing the bus", - Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + - "it reconnects, what it says and what it is asked waits.", - Resolved: "Resolved: the client stays connected"}, - "core-unhealthy": {Headline: "The bus is not healthy", - Needs: "check the machine the bus runs on; the details say what fails.", - Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + - "calls may fail until it recovers.", - Resolved: "The bus is healthy again"}, - "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", - Needs: "restart the watchman if this stays, from the mesh MCP server.", - Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + - "signals are not noticed.", - Resolved: "The watchdogs run again"}, - "probe-failed": {Headline: "A check of the mesh could not run", - Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + - "every few minutes.", - Resolved: "The check runs again"}, -} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 71c5d5d5b..1ca997746 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -17,6 +17,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-sdk/go/buscompose" ) @@ -139,7 +140,7 @@ func (c *Checker) streams(stream string) ([]streamRow, error) { } // probeStreams is D7: every stream the mesh defines exists as defined. -func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { +func probeStreams(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { rows, err := c.streams("") if err != nil { return nil, err @@ -148,17 +149,17 @@ func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { for _, r := range rows { held[r.Name] = r } - var out []Observation + var out []meshconditionsclient.Observation for _, s := range buscompose.MeshStreams() { have, ok := held[s.Name] if !ok { - out = append(out, Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + out = append(out, meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: s.Name, Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), Said: "no stream " + s.Name}) continue } if differs := streamDiffers(s, have); differs != "" { - out = append(out, Observation{Ref: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + out = append(out, meshconditionsclient.Observation{Ref: "stream-redefined", Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: s.Name, Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), Said: differs}) } @@ -212,7 +213,7 @@ type closedAnswer struct { } // probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. -func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { +func probeReconnects(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) if err != nil { return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) @@ -225,7 +226,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { if hours <= 0 { hours = 1 } - var out []Observation + var out []meshconditionsclient.Observation for _, u := range read.Users { if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { continue // refused before it logged in is nobody's reconnect loop @@ -253,8 +254,8 @@ func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { partial = " (at least: the server's record of closed connections does not reach back the whole hour)" } who, machine := busUserWords(u.User) - out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning, - Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, + out = append(out, meshconditionsclient.Observation{Kind: "bus-reconnects", Severity: meshconditionsclient.Warning, + Scope: meshconditionsclient.ScopeBus, ID: u.User, Machine: machine, Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, @@ -289,9 +290,9 @@ func busUserWords(user string) (string, string) { // probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the // watchman's pulse on `checked`, is the other half. -func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { +func probeWatchdogs(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { - return []Observation{{Ref: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + return []meshconditionsclient.Observation{{Ref: "silent", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeCore, ID: "watchdogs", Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + "late signal is being said", Said: firstLine(err.Error())}}, nil @@ -303,7 +304,7 @@ func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { // machine and back. The mesh defines no durable consumer of its own since the controller ended: the account consumers // are the grants part's to make and judge (issue 1126). The question to the bus module is that request: an answer is its round trip, // and no answer is no pass. -func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { +func probeBusHealth(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { rows, err := c.streams("") if err != nil { return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) @@ -321,12 +322,12 @@ func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { if len(problems) == 0 { return nil, nil } - return []Observation{{Ref: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + return []meshconditionsclient.Observation{{Ref: "unhealthy", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "mesh", Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil } // sorted is findings in a stated order, so two runs over one mesh say the same. -func sorted(out []Observation) []Observation { +func sorted(out []meshconditionsclient.Observation) []meshconditionsclient.Observation { sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) return out } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go index 406d0d051..22f8658a0 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings" "git.novox.be/novox/mesh-catalog/modules/restic/backupgood" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -191,7 +192,7 @@ func (c *Checker) dataFacts(ctx context.Context) (dataFacts, error) { } // probeData is D13. -func probeData(ctx context.Context, c *Checker) ([]Observation, error) { +func probeData(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { f, err := c.dataFacts(ctx) if err != nil { return nil, err @@ -201,9 +202,9 @@ func probeData(ctx context.Context, c *Checker) ([]Observation, error) { func severityOf(class string) string { if class == catalogue.ClassIrreplaceable { - return Urgent + return meshconditionsclient.Urgent } - return Warning + return meshconditionsclient.Warning } func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item } @@ -262,14 +263,14 @@ func firstSeen(r diskreadings.Reading, now time.Time) time.Time { return now } -func obs(kind string, id, machine string, sev string, summary, said string) Observation { - return Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said, - Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}} +func obs(kind string, id, machine string, sev string, summary, said string) meshconditionsclient.Observation { + return meshconditionsclient.Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said, + Scope: ScopeMachine, ID: id, Machine: machine} } // dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was. -func dataFindings(f dataFacts) []Observation { - var out []Observation +func dataFindings(f dataFacts) []meshconditionsclient.Observation { + var out []meshconditionsclient.Observation arrays := map[string][]diskreadings.Reading{} type shrunk struct { machine, dataset, class string @@ -280,9 +281,8 @@ func dataFindings(f dataFacts) []Observation { byItem := map[string][]diskreadings.Reading{} retiredCopies := map[string][]diskreadings.Reading{} for _, m := range f.unmeasured { - o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+ + o := obs(kindDataUnmeasured, m, m, meshconditionsclient.Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+ "declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data") - o.Confirm = true out = append(out, o) } rs := append([]diskreadings.Reading(nil), f.readings...) @@ -296,7 +296,7 @@ func dataFindings(f dataFacts) []Observation { if !r.RetiredAt.IsZero() { retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r) if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) { - out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning, + out = append(out, obs(kindCleanupWaiting, id, r.Node, meshconditionsclient.Warning, fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+ "`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again", r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24), @@ -428,7 +428,7 @@ func dataFindings(f dataFacts) []Observation { fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ "an empty replacement of its data. Move the data, or assign it back where its data is", a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "") - ob.Subject.Also = []string{o.Node} + ob.Also = []string{o.Node} out = append(out, ob) break } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go index edeaba270..b4d96dcca 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go @@ -14,6 +14,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-sdk/go/buscompose" ) @@ -54,25 +55,25 @@ func mibWords(b uint64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1024 // fillKey is where a bucket's or log's fill finding is kept. func fillKey(stream string) string { - return Observation{Subject: Subject{Scope: ScopeBus, ID: stream}, Ref: "filling"}.Key() + return meshconditionsclient.Observation{Scope: meshconditionsclient.ScopeBus, ID: stream, Ref: "filling"}.Key() } // fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and, while its finding // is written, at fillClearBelow or above too; nothing otherwise, which is what clears it. -func fillObservation(stream string, r fillRow, open bool) (Observation, bool) { +func fillObservation(stream string, r fillRow, open bool) (meshconditionsclient.Observation, bool) { module, kind, name, ok := bucketOrLog(stream) if !ok || r.MaxBytes == 0 { - return Observation{}, false // no module's, or with no cap: it cannot fill + return meshconditionsclient.Observation{}, false // no module's, or with no cap: it cannot fill } // Compared in bytes, not rounded percent: 74.9% is not 75%. atRaise := r.Bytes*100 >= r.MaxBytes*fillRaiseAt aboveClear := r.Bytes*100 >= r.MaxBytes*fillClearBelow if !atRaise && !(open && aboveClear) { - return Observation{}, false + return meshconditionsclient.Observation{}, false } pct := r.Bytes * 100 / r.MaxBytes - return Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: Warning, - Subject: Subject{Scope: ScopeBus, ID: stream}, + return meshconditionsclient.Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: meshconditionsclient.Warning, + Scope: meshconditionsclient.ScopeBus, ID: stream, Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write", module, kind, name, mibWords(r.Bytes), mibWords(r.MaxBytes), pct), Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes, @@ -86,7 +87,7 @@ func fillObservation(stream string, r fillRow, open bool) (Observation, bool) { // probeFill is D-fill: every module's bucket and log on the bus judged against its cap. The bus module not answering // is the probe failing to run, which clears nothing; a finding already written counts as open. -func probeFill(_ context.Context, c *Checker) ([]Observation, error) { +func probeFill(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { raw, err := c.Ask(StreamsTool, map[string]any{}) if err != nil { return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) @@ -102,7 +103,7 @@ func probeFill(_ context.Context, c *Checker) ([]Observation, error) { return nil, fmt.Errorf("its own findings cannot be read, so a fill between the thresholds cannot be judged: %w", err) } sort.Slice(a.Detail, func(i, j int) bool { return a.Detail[i].Name < a.Detail[j].Name }) - var out []Observation + var out []meshconditionsclient.Observation for _, r := range a.Detail { if o, said := fillObservation(r.Name, r, written.has(fillKey(r.Name))); said { out = append(out, o) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go index e2fe114ce..b0fd1d722 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go @@ -10,9 +10,6 @@ import ( "context" "encoding/json" "fmt" - watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" - assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" - placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "net" "regexp" "slices" @@ -20,6 +17,10 @@ import ( "strings" "sync" + watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" + assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" + placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" ) @@ -127,7 +128,7 @@ var lookupHost = func(ctx context.Context, host string) ([]string, error) { // probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are every machine's // private address and the address its endpoint names, from placement; the ban lists are asked of each machine's holder // of node-intrusion-prevention that is heard from. -func probeBans(ctx context.Context, c *Checker) ([]Observation, error) { +func probeBans(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { places, err := c.placements() if err != nil { return nil, err @@ -182,7 +183,7 @@ func probeBans(ctx context.Context, c *Checker) ([]Observation, error) { }(i, node) } wg.Wait() - var out []Observation + var out []meshconditionsclient.Observation var unasked []string for i, node := range holders { if errs[i] != nil { @@ -205,8 +206,8 @@ func probeBans(ctx context.Context, c *Checker) ([]Observation, error) { } } if len(banned) > 0 { - out = append(out, Observation{Kind: "own-address-banned", Ref: "bans-the-mesh", Severity: Urgent, - Subject: Subject{Scope: ScopeMachine, ID: node, Machine: node}, + out = append(out, meshconditionsclient.Observation{Kind: "own-address-banned", Ref: "bans-the-mesh", Severity: meshconditionsclient.Urgent, + Scope: ScopeMachine, ID: node, Machine: node, Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es) — %s: the mesh is locked out "+ "of itself there (ADR 0186)", node, len(banned), strings.Join(whose, ", ")), Said: strings.Join(banned, ", "), diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go index ee20deaa9..eee21cc54 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go @@ -9,13 +9,15 @@ package main import ( "context" "fmt" - engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "sort" "strings" "sync" "time" + + engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" ) // The engine's words and bounds the judgement reads (mesh-host rootsearch). @@ -203,7 +205,7 @@ func (c *Checker) confined(node string, place placing.Placement, heard bool) (co // probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest // statement, unable to become root without a person (ADR 0266). -func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) { +func probeAgentAccounts(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { places, err := c.placements() if err != nil { return nil, err @@ -217,7 +219,7 @@ func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) { nodes = append(nodes, n) } sort.Strings(nodes) - var out []Observation + var out []meshconditionsclient.Observation for _, n := range nodes { if places[n].Decided.AgentAccount == "" { continue @@ -231,8 +233,8 @@ func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) { if v.Confined || v.Quiet { continue } - out = append(out, Observation{Kind: "agent-can-become-root", Ref: "agent-root", Severity: Urgent, - Subject: Subject{Scope: ScopeMachine, ID: n, Machine: n}, + out = append(out, meshconditionsclient.Observation{Kind: "agent-can-become-root", Ref: "agent-root", Severity: meshconditionsclient.Urgent, + Scope: ScopeMachine, ID: n, Machine: n, Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+ "answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why), Said: v.Why, diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index ee1348e7d..3a51c5670 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -4,17 +4,19 @@ import ( "context" "encoding/json" "errors" + "slices" + "strings" + "testing" + "time" + engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "slices" - "strings" - "testing" - "time" "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings" "git.novox.be/novox/mesh-catalog/modules/restic/backupgood" @@ -105,7 +107,7 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) { return nil, errors.New("nats: no responders available for request") } -func keysOf(found []Observation) []string { +func keysOf(found []meshconditionsclient.Observation) []string { var out []string for _, f := range found { out = append(out, f.Key()) @@ -117,7 +119,7 @@ func keysOf(found []Observation) []string { func TestTheBusProbes(t *testing.T) { cases := []struct { name string - probe func(context.Context, *Checker) ([]Observation, error) + probe func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) bus bus want []string err string @@ -305,9 +307,9 @@ func placed(s *memStore) map[string]placing.Placement { // filling is a store holding the self-check's written fill finding of a stream: the condition open. func filling(stream string) *memStore { s := newStore() - o := Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream}, + o := meshconditionsclient.Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: stream, Summary: "filling", Source: "D-fill"} - raw, _ := json.Marshal(o.Finding("run", time.Now())) + raw, _ := json.Marshal(o.Finding(Module, "run", time.Now())) _ = s.Put(FindingsState, o.Key(), json.RawMessage(raw)) return s } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go index fad166b4c..ba9ac720a 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" meshselfcheck "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck" ) @@ -94,7 +95,7 @@ type Probe struct { // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): // answered deferred, never a pass. Deferred string - Run func(ctx context.Context, c *Checker) ([]Observation, error) + Run func(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) } // ProbeVerdict, Counts and Run are a run as the seat mesh-self-check's contract holds it: the body of `checked`, the controller's @@ -143,7 +144,7 @@ func (c *Checker) LastRun() *Run { } type result struct { - found []Observation + found []meshconditionsclient.Observation err error took time.Duration } @@ -178,9 +179,9 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { written, readErr := c.written() seen := map[string]bool{} - want := map[string]Observation{} // every finding this run writes, by key - ran := map[string]bool{} // the sources whose findings this run judged whole - var blind []Observation + want := map[string]meshconditionsclient.Observation{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []meshconditionsclient.Observation for i, p := range c.Registry { if !on(p.ID) { // Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no @@ -198,8 +199,8 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { case r.err != nil: v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() run.Counts.FailedToRun++ - blind = append(blind, Observation{Kind: "probe-failed", Ref: "failed", Severity: Warning, Confirm: true, - Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, + blind = append(blind, meshconditionsclient.Observation{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning, + Scope: meshconditionsclient.ScopeProbe, ID: p.ID, Source: sourceSelf, Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), Said: firstLine(r.err.Error())}) default: @@ -212,7 +213,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { f = worded(f) key := f.Key() seen[key] = true - if f.Confirm && !c.seenBefore(key) && !written.has(key) { + if confirm(f) && !c.seenBefore(key) && !written.has(key) { v.Unconfirmed = append(v.Unconfirmed, f.Summary) continue } @@ -321,7 +322,7 @@ func (c *Checker) written() (writtenFindings, error) { // write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding // of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). -func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error { +func (c *Checker) write(written writtenFindings, want map[string]meshconditionsclient.Observation, ran map[string]bool, run string) error { var errs []string keys := make([]string, 0, len(want)) for k := range want { @@ -330,8 +331,8 @@ func (c *Checker) write(written writtenFindings, want map[string]Observation, ra sort.Strings(keys) now := c.Now() for _, k := range keys { - f := want[k].Finding(run, now) - if was, ok := written[k]; ok && says(was) == says(f) { + f := want[k].Finding(Module, run, now) + if was, ok := written[k]; ok && meshconditionsclient.Says(was) == meshconditionsclient.Says(f) { continue } raw, err := json.Marshal(f) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index ce46ad85e..f615b3cc2 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -5,13 +5,15 @@ import ( "encoding/json" "errors" "fmt" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "slices" "sort" "strings" "sync" "testing" "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" ) // memStore is the bus's buckets in memory, counting the writes and the reads one key at a time, and handing every @@ -113,29 +115,29 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { // could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can // be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). func TestARunWritesFindingsAsTheyChange(t *testing.T) { - var found []Observation + var found []meshconditionsclient.Observation var fail error - steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }} + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { return found, fail }} deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} - missing := Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + missing := meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "EVENTS", Summary: "the stream EVENTS is not on the bus"} - behind := Observation{Kind: "bus-reconnects", Ref: "behind", Severity: Warning, Confirm: true, - Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} + behind := meshconditionsclient.Observation{Kind: kindDataUnmeasured, Ref: "behind", Severity: meshconditionsclient.Warning, + Scope: meshconditionsclient.ScopeBus, ID: "EVENTS.controller", Summary: "far behind"} store := newStore() c, said := checker(store, steady, deferred) steps := []struct { name string - found []Observation + found []meshconditionsclient.Observation fail error want []string // finding keys written after the run verdict string missingPs int // puts of the missing finding so far }{ - {"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"seen twice: written", []Observation{missing, behind}, nil, + {"a finding is written", []meshconditionsclient.Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []meshconditionsclient.Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []meshconditionsclient.Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []meshconditionsclient.Observation{missing, behind}, nil, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, {"blind once: clears nothing, held", nil, errors.New("timed out"), []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, @@ -187,7 +189,7 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { t.Errorf("the last verdict counts %+v", r.Counts) } // A finding carries the operator's words of its kind, the probe that found it and who observed it. - found, fail = []Observation{missing}, nil + found, fail = []meshconditionsclient.Observation{missing}, nil c.RunOnce(context.Background(), "the test") raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") f, err := decodeFinding(raw) @@ -210,9 +212,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { within := Within Within = 50 * time.Millisecond t.Cleanup(func() { Within = within }) - for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){ - "panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") }, - "overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil }, + for name, run := range map[string]func(context.Context, *Checker) ([]meshconditionsclient.Observation, error){ + "panic": func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]meshconditionsclient.Observation, error) { + time.Sleep(time.Second) + return nil, nil + }, } { c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) c.Now = time.Now @@ -227,12 +232,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { // every state the module keeps (ADR 0564 §12). func TestTheVerbsAndReads(t *testing.T) { store := newStore() - many := make([]Observation, 25) + many := make([]meshconditionsclient.Observation, 25) for i := range many { - many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + many[i] = meshconditionsclient.Observation{Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: fmt.Sprintf("S%02d", i), Summary: "x"} } c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", - Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }}) + Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { return many, nil }}) if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { t.Fatalf("before a run: %v", err) } @@ -278,11 +283,11 @@ func TestTheVerbsAndReads(t *testing.T) { // and is reported by the verb as off by setting; the default list is D13 and D-fill (the operator, 2026-10-30). func TestOnlyTheProbesTheSettingListsRun(t *testing.T) { ran := map[string]int{} - finds := Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "gone"} + finds := meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "EVENTS", Summary: "gone"} probe := func(id string) Probe { - return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { + return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { ran[id]++ - return []Observation{finds}, nil + return []meshconditionsclient.Observation{finds}, nil }} } store := newStore() diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go index d932f93fc..73c1b96e5 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -26,6 +26,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-sdk/go/testbus" ) @@ -295,7 +296,7 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { t.Fatalf("the missing stream is not a finding on the bus: %v", err) } f, err := decodeFinding(e.Value()) - if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + if err != nil || f.Kind != "stream-wrong" || f.Severity != meshconditionsclient.Urgent || f.Source != "D7" { t.Errorf("the finding is %s", e.Value()) } verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/words.go b/modules/mesh-self-check/cmd/mesh-self-check/words.go new file mode 100644 index 000000000..dcf0d904d --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/words.go @@ -0,0 +1,62 @@ +package main + +// The operator's words for the kinds the self-check raises; the observation and its finding are the conditions +// client's (novox/hq ADR 0564 §2, ADR 0634 §4). + +import "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + +// worded gives a finding the operator's words registered for its kind, where it says none of its own. +func worded(f meshconditionsclient.Observation) meshconditionsclient.Observation { + w, ok := words[f.Kind] + if !ok { + return f + } + if f.Headline == "" { + f.Headline = w.Headline + } + if f.Explanation == "" { + f.Explanation = w.Explanation + } + if f.Resolved == "" { + f.Resolved = w.Resolved + } + if f.Needs == "" { + f.Needs = w.Needs + } + return f +} + +type plain struct{ Headline, Explanation, Resolved, Needs string } + +// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the +// controller's name for itself replaced by the self-check's where the words spoke of it. +var words = map[string]plain{ + "stream-wrong": {Headline: "Part of the bus's storage is wrong", + Needs: "check the machine the bus runs on; the details say what is missing.", + Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", + Resolved: "The bus's storage is right again"}, + "bus-reconnects": {Headline: "A client keeps losing the bus", + Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + + "it reconnects, what it says and what it is asked waits.", + Resolved: "Resolved: the client stays connected"}, + "core-unhealthy": {Headline: "The bus is not healthy", + Needs: "check the machine the bus runs on; the details say what fails.", + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + + "calls may fail until it recovers.", + Resolved: "The bus is healthy again"}, + "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", + Needs: "restart the watchman if this stays, from the mesh MCP server.", + Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + + "signals are not noticed.", + Resolved: "The watchdogs run again"}, + "probe-failed": {Headline: "A check of the mesh could not run", + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + + "every few minutes.", + Resolved: "The check runs again"}, +} + +// confirm says one look can be wrong about the finding: it is written only when two runs in a row see it (issue 277). +// The kinds are the self-check's own: a probe that could not run, and a holder that measured nothing of the data. +func confirm(o meshconditionsclient.Observation) bool { + return o.Kind == "probe-failed" || o.Kind == kindDataUnmeasured +}