novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

mesh-healers and mesh-self-check use the conditions client's Observation, Finding encoding and Says, not copies of their own (ADR 0634 §4; issue 1966)

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-05 17:05:27 +01:00
parent 1f3a293988
commit 2978dd5bed
17 changed files with 195 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-healers/cmd/mesh-healers/healers.go
+15 -15
View File
@@ -98,12 +98,12 @@ var ErrExists = errors.New("the key was made by another writer first")
// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside.
type Condition struct {
Key string `json:"key"`
Kind string `json:"kind"`
Subject Subject `json:"subject"`
Severity string `json:"severity"`
Resolver string `json:"resolver"`
Tried []json.RawMessage `json:"tried,omitempty"`
Key string `json:"key"`
Kind string `json:"kind"`
Subject meshconditions.Subject `json:"subject"`
Severity string `json:"severity"`
Resolver string `json:"resolver"`
Tried []json.RawMessage `json:"tried,omitempty"`
}
// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until
@@ -240,7 +240,7 @@ func (h *Healers) Judge(c Condition, by string) (string, error) {
h.see("blind", nil)
acts := actsWithin(all, now.Add(-BrakeWindow))
if braked, said := h.brakeHolds(acts, now); braked {
h.see("brake", []Observation{brakeFinding(acts, said)})
h.see("brake", []meshconditionsclient.Observation{brakeFinding(acts, said)})
return "every healer has stopped: " + said, nil
}
h.see("brake", nil)
@@ -483,10 +483,10 @@ func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) {
return false, said
}
var brakeKey = meshconditionsclient.Key(ScopeMesh, "healers", "braked")
var brakeKey = meshconditionsclient.Key(meshconditionsclient.ScopeMesh, "healers", "braked")
// brakeFinding is the brake, as the urgent finding that says it.
func brakeFinding(acts []Attempt, said string) Observation {
func brakeFinding(acts []Attempt, said string) meshconditionsclient.Observation {
counts := map[string]int{}
for _, a := range acts {
counts[a.Healer+" on "+a.Condition]++
@@ -496,7 +496,7 @@ func brakeFinding(acts []Attempt, said string) Observation {
most = append(most, fmt.Sprintf("%s ×%d", what, n))
}
sort.Strings(most)
return Observation{Kind: "healers-braked", Ref: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"},
return meshconditionsclient.Observation{Kind: "healers-braked", Ref: "braked", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeMesh, ID: "healers",
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
"healer looping, not repairing — " + said,
Said: strings.Join(most, ", "),
@@ -509,8 +509,8 @@ func brakeFinding(acts []Attempt, said string) Observation {
// blind writes the healers' own blindness: they cannot read what they did.
func (h *Healers) blind(err error) {
h.see("blind", []Observation{{Kind: "probe-failed", Ref: "failed", Severity: Warning,
Subject: Subject{Scope: ScopeProbe, ID: "healers"},
h.see("blind", []meshconditionsclient.Observation{{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeProbe, ID: "healers",
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
Said: firstLine(err.Error()), Headline: "A check of the mesh could not run",
Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.",
@@ -519,12 +519,12 @@ func (h *Healers) blind(err error) {
// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it
// no longer sees (ADR 0564 §2).
func (h *Healers) see(source string, found []Observation) {
func (h *Healers) see(source string, found []meshconditionsclient.Observation) {
now := h.Now()
want := map[string]meshconditions.FindingsByKeyValue{}
for _, o := range found {
o.Source = source
want[o.Key()] = o.Finding("", now)
want[o.Key()] = o.Finding(Module, "", now)
}
keys, err := h.Store.Keys(FindingsState)
if err != nil {
@@ -542,7 +542,7 @@ func (h *Healers) see(source string, found []Observation) {
continue
}
if f, still := want[k]; still {
if says(was) == says(f) {
if meshconditionsclient.Says(was) == meshconditionsclient.Says(f) {
delete(want, k)
}
continue
modules/mesh-healers/cmd/mesh-healers/healers_test.go
+6 -4
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"errors"
"fmt"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"slices"
"sort"
"strings"
@@ -12,6 +11,9 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
)
@@ -171,7 +173,7 @@ var drill = Healer{ID: "HD", Kinds: []string{"drill-broken"}, Condition: "a dril
// lost is a condition the drill healer answers, on a module.
func lost(module string) Condition {
return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "drill-broken", Severity: "warning", Resolver: "self",
Subject: Subject{Scope: "bus", ID: "EVENTS." + module}}
Subject: meshconditions.Subject{Scope: "bus", ID: "EVENTS." + module}}
}
func body(c Condition) []byte {
@@ -288,7 +290,7 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
raw, braked, _ := b.store.Get(FindingsState, brakeKey)
var f meshconditions.FindingsByKeyValue
_ = json.Unmarshal(raw, &f)
if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module {
if !braked || f.Severity != meshconditionsclient.Urgent || f.Kind != "healers-braked" || f.Observer != Module {
t.Errorf("the brake is written as %s", raw)
}
b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held
@@ -319,7 +321,7 @@ func TestTheRegistryActsOnNothingHere(t *testing.T) {
w := newWorld()
w.h.Registry = Registry()
for _, kind := range []string{"consumer-lost", "holder-silent", "consumer-behind"} {
c := Condition{Key: "bus.EVENTS.records." + kind, Kind: kind, Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}
c := Condition{Key: "bus.EVENTS.records." + kind, Kind: kind, Subject: meshconditions.Subject{Scope: "bus", ID: "EVENTS.records"}}
if said, _ := w.h.Judge(c, ""); len(w.asked) != 0 || !strings.Contains(said, "no healer here") {
t.Errorf("%s: %q, asked %v", kind, said, w.asked)
}
modules/mesh-healers/cmd/mesh-healers/observation.go
-78
View File
@@ -1,78 +0,0 @@
package main
// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their
// blindness and S15, each written into the bucket `findings` as the seat mesh-conditions' finding, whose key is the
// controller's key for the same condition.
import (
"encoding/json"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// The severities and scopes, as mesh-sdk's health names them.
const (
Urgent = meshconditionsclient.Urgent
Warning = meshconditionsclient.Warning
ScopeProbe = meshconditionsclient.ScopeProbe
ScopeMesh = meshconditionsclient.ScopeMesh
)
// Subject is what a finding is about.
type Subject = meshconditions.Subject
// Observation is one thing a probe sees wrong now.
type Observation struct {
Kind string
Ref string
Subject Subject
Severity string
Summary string
Said string
Headline string
Explanation string
Resolved string
Needs string
// Source is what saw it: the brake, the healers' blindness, or S15.
Source string
}
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
func (o Observation) Key() string {
token := o.Ref
if token == "" {
token = o.Kind
}
return meshconditionsclient.Key(o.Subject.Scope, o.Subject.ID, token)
}
// Finding is the observation as written by run at, its evidence the one line it said.
func (o Observation) Finding(run string, at time.Time) meshconditions.FindingsByKeyValue {
f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
Observer: Module, Run: run, At: at.UTC(), Evidence: []meshconditions.Evidence{}}
if o.Ref != "" && o.Ref != o.Kind {
f.Ref = o.Ref
}
said := o.Said
if said == "" {
said = o.Summary
}
f.Evidence = append(f.Evidence, meshconditions.Evidence{At: at.UTC(), Said: said})
return f
}
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
// (ADR 0289 §1).
func says(f meshconditions.FindingsByKeyValue) string {
f.At, f.Run, f.V = time.Time{}, "", 0
ev := make([]meshconditions.Evidence, len(f.Evidence))
for i, e := range f.Evidence {
ev[i] = meshconditions.Evidence{Said: e.Said}
}
f.Evidence = ev
raw, _ := json.Marshal(f)
return string(raw)
}
modules/mesh-healers/cmd/mesh-healers/wanted.go
+5 -4
View File
@@ -16,6 +16,7 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
)
@@ -107,7 +108,7 @@ func (h *Healers) Seeded(answer []byte) (int, error) {
// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more,
// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight.
func (h *Healers) Wanted() []Observation {
func (h *Healers) Wanted() []meshconditionsclient.Observation {
h.mu.Lock()
defer h.mu.Unlock()
now := h.Now()
@@ -125,7 +126,7 @@ func (h *Healers) Wanted() []Observation {
}
}
sort.Strings(causes)
var found []Observation
var found []meshconditionsclient.Observation
var next time.Time
for _, cause := range causes {
acts := byCause[cause]
@@ -144,8 +145,8 @@ func (h *Healers) Wanted() []Observation {
}
}
words := strings.NewReplacer(".", " ", "_", " ").Replace(cause)
found = append(found, Observation{Kind: "healer-wanted", Ref: "healer-wanted", Severity: Warning,
Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause},
found = append(found, meshconditionsclient.Observation{Kind: "healer-wanted", Ref: "healer-wanted", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeMesh, ID: "hand-acts." + cause,
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts),
int(Within.Hours()/24), newest.By, wanted),
Said: strings.Join(lines, "; "),
modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go
+2 -1
View File
@@ -15,7 +15,6 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"io"
"os"
"os/exec"
@@ -25,6 +24,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
modules/mesh-self-check/cmd/mesh-self-check/main.go
+3 -2
View File
@@ -9,11 +9,12 @@ import (
"context"
"encoding/json"
"fmt"
assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"os"
"time"
assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
modules/mesh-self-check/cmd/mesh-self-check/observation.go
-133
View File
@@ -1,133 +0,0 @@
package main
// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an
// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong
// about it. It is written into the bucket `findings` as the seat mesh-conditions' finding, whose key is the controller's key
// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key.
import (
"encoding/json"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// The severities and scopes, as mesh-sdk's health names them.
const (
Urgent = meshconditionsclient.Urgent
Warning = meshconditionsclient.Warning
ScopeBus = meshconditionsclient.ScopeBus
ScopeCore = meshconditionsclient.ScopeCore
ScopeProbe = meshconditionsclient.ScopeProbe
ScopeMesh = meshconditionsclient.ScopeMesh
)
// Subject is what a finding is about.
type Subject = meshconditions.Subject
// Observation is one thing a probe sees wrong now.
type Observation struct {
Kind string
Ref string
Subject Subject
Severity string
Summary string
Said string
Headline string
Explanation string
Resolved string
Needs string
// Source is the probe that saw it, or the self-check itself for a probe that could not run.
Source string
// Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277).
Confirm bool
}
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
func (o Observation) Key() string {
token := o.Ref
if token == "" {
token = o.Kind
}
return meshconditionsclient.Key(o.Subject.Scope, o.Subject.ID, token)
}
// Finding is the observation as written by run at, its evidence the one line it said.
func (o Observation) Finding(run string, at time.Time) meshconditions.FindingsByKeyValue {
f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
Observer: Module, Run: run, At: at.UTC(), Evidence: []meshconditions.Evidence{}}
if o.Ref != "" && o.Ref != o.Kind {
f.Ref = o.Ref
}
said := o.Said
if said == "" {
said = o.Summary
}
f.Evidence = append(f.Evidence, meshconditions.Evidence{At: at.UTC(), Said: said})
return f
}
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
// (ADR 0289 §1).
func says(f meshconditions.FindingsByKeyValue) string {
f.At, f.Run, f.V = time.Time{}, "", 0
ev := make([]meshconditions.Evidence, len(f.Evidence))
for i, e := range f.Evidence {
ev[i] = meshconditions.Evidence{Said: e.Said}
}
f.Evidence = ev
raw, _ := json.Marshal(f)
return string(raw)
}
// worded gives a finding the operator's words registered for its kind, where it says none of its own.
func worded(f Observation) Observation {
w, ok := words[f.Kind]
if !ok {
return f
}
if f.Headline == "" {
f.Headline = w.Headline
}
if f.Explanation == "" {
f.Explanation = w.Explanation
}
if f.Resolved == "" {
f.Resolved = w.Resolved
}
if f.Needs == "" {
f.Needs = w.Needs
}
return f
}
type plain struct{ Headline, Explanation, Resolved, Needs string }
// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the
// controller's name for itself replaced by the self-check's where the words spoke of it.
var words = map[string]plain{
"stream-wrong": {Headline: "Part of the bus's storage is wrong",
Needs: "check the machine the bus runs on; the details say what is missing.",
Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
Resolved: "The bus's storage is right again"},
"bus-reconnects": {Headline: "A client keeps losing the bus",
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " +
"it reconnects, what it says and what it is asked waits.",
Resolved: "Resolved: the client stays connected"},
"core-unhealthy": {Headline: "The bus is not healthy",
Needs: "check the machine the bus runs on; the details say what fails.",
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " +
"calls may fail until it recovers.",
Resolved: "The bus is healthy again"},
"watchdogs-silent": {Headline: "The mesh's watchdogs stopped",
Needs: "restart the watchman if this stays, from the mesh MCP server.",
Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " +
"signals are not noticed.",
Resolved: "The watchdogs run again"},
"probe-failed": {Headline: "A check of the mesh could not run",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " +
"every few minutes.",
Resolved: "The check runs again"},
}
modules/mesh-self-check/cmd/mesh-self-check/probes.go
+14 -13
View File
@@ -17,6 +17,7 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
@@ -139,7 +140,7 @@ func (c *Checker) streams(stream string) ([]streamRow, error) {
}
// probeStreams is D7: every stream the mesh defines exists as defined.
func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) {
func probeStreams(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
rows, err := c.streams("")
if err != nil {
return nil, err
@@ -148,17 +149,17 @@ func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) {
for _, r := range rows {
held[r.Name] = r
}
var out []Observation
var out []meshconditionsclient.Observation
for _, s := range buscompose.MeshStreams() {
have, ok := held[s.Name]
if !ok {
out = append(out, Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name},
out = append(out, meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: s.Name,
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
Said: "no stream " + s.Name})
continue
}
if differs := streamDiffers(s, have); differs != "" {
out = append(out, Observation{Ref: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name},
out = append(out, meshconditionsclient.Observation{Ref: "stream-redefined", Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: s.Name,
Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs),
Said: differs})
}
@@ -212,7 +213,7 @@ type closedAnswer struct {
}
// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour.
func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
func probeReconnects(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1})
if err != nil {
return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err)
@@ -225,7 +226,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
if hours <= 0 {
hours = 1
}
var out []Observation
var out []meshconditionsclient.Observation
for _, u := range read.Users {
if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax {
continue // refused before it logged in is nobody's reconnect loop
@@ -253,8 +254,8 @@ func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
}
who, machine := busUserWords(u.User)
out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning,
Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine},
out = append(out, meshconditionsclient.Observation{Kind: "bus-reconnects", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeBus, ID: u.User, Machine: machine,
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+
"reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax),
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed,
@@ -289,9 +290,9 @@ func busUserWords(user string) (string, string) {
// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the
// watchman's pulse on `checked`, is the other half.
func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) {
func probeWatchdogs(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil {
return []Observation{{Ref: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"},
return []meshconditionsclient.Observation{{Ref: "silent", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeCore, ID: "watchdogs",
Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " +
"late signal is being said",
Said: firstLine(err.Error())}}, nil
@@ -303,7 +304,7 @@ func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) {
// machine and back. The mesh defines no durable consumer of its own since the controller ended: the account consumers
// are the grants part's to make and judge (issue 1126). The question to the bus module is that request: an answer is its round trip,
// and no answer is no pass.
func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) {
func probeBusHealth(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
rows, err := c.streams("")
if err != nil {
return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err)
@@ -321,12 +322,12 @@ func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) {
if len(problems) == 0 {
return nil, nil
}
return []Observation{{Ref: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"},
return []meshconditionsclient.Observation{{Ref: "unhealthy", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "mesh",
Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil
}
// sorted is findings in a stated order, so two runs over one mesh say the same.
func sorted(out []Observation) []Observation {
func sorted(out []meshconditionsclient.Observation) []meshconditionsclient.Observation {
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
return out
}
modules/mesh-self-check/cmd/mesh-self-check/probes_data.go
+12 -12
View File
@@ -21,6 +21,7 @@ import (
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
@@ -191,7 +192,7 @@ func (c *Checker) dataFacts(ctx context.Context) (dataFacts, error) {
}
// probeData is D13.
func probeData(ctx context.Context, c *Checker) ([]Observation, error) {
func probeData(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
f, err := c.dataFacts(ctx)
if err != nil {
return nil, err
@@ -201,9 +202,9 @@ func probeData(ctx context.Context, c *Checker) ([]Observation, error) {
func severityOf(class string) string {
if class == catalogue.ClassIrreplaceable {
return Urgent
return meshconditionsclient.Urgent
}
return Warning
return meshconditionsclient.Warning
}
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
@@ -262,14 +263,14 @@ func firstSeen(r diskreadings.Reading, now time.Time) time.Time {
return now
}
func obs(kind string, id, machine string, sev string, summary, said string) Observation {
return Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said,
Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}}
func obs(kind string, id, machine string, sev string, summary, said string) meshconditionsclient.Observation {
return meshconditionsclient.Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said,
Scope: ScopeMachine, ID: id, Machine: machine}
}
// dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was.
func dataFindings(f dataFacts) []Observation {
var out []Observation
func dataFindings(f dataFacts) []meshconditionsclient.Observation {
var out []meshconditionsclient.Observation
arrays := map[string][]diskreadings.Reading{}
type shrunk struct {
machine, dataset, class string
@@ -280,9 +281,8 @@ func dataFindings(f dataFacts) []Observation {
byItem := map[string][]diskreadings.Reading{}
retiredCopies := map[string][]diskreadings.Reading{}
for _, m := range f.unmeasured {
o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+
o := obs(kindDataUnmeasured, m, m, meshconditionsclient.Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+
"declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data")
o.Confirm = true
out = append(out, o)
}
rs := append([]diskreadings.Reading(nil), f.readings...)
@@ -296,7 +296,7 @@ func dataFindings(f dataFacts) []Observation {
if !r.RetiredAt.IsZero() {
retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r)
if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) {
out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning,
out = append(out, obs(kindCleanupWaiting, id, r.Node, meshconditionsclient.Warning,
fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24),
@@ -428,7 +428,7 @@ func dataFindings(f dataFacts) []Observation {
fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
"an empty replacement of its data. Move the data, or assign it back where its data is",
a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "")
ob.Subject.Also = []string{o.Node}
ob.Also = []string{o.Node}
out = append(out, ob)
break
}
modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go
+9 -8
View File
@@ -14,6 +14,7 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
@@ -54,25 +55,25 @@ func mibWords(b uint64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1024
// fillKey is where a bucket's or log's fill finding is kept.
func fillKey(stream string) string {
return Observation{Subject: Subject{Scope: ScopeBus, ID: stream}, Ref: "filling"}.Key()
return meshconditionsclient.Observation{Scope: meshconditionsclient.ScopeBus, ID: stream, Ref: "filling"}.Key()
}
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and, while its finding
// is written, at fillClearBelow or above too; nothing otherwise, which is what clears it.
func fillObservation(stream string, r fillRow, open bool) (Observation, bool) {
func fillObservation(stream string, r fillRow, open bool) (meshconditionsclient.Observation, bool) {
module, kind, name, ok := bucketOrLog(stream)
if !ok || r.MaxBytes == 0 {
return Observation{}, false // no module's, or with no cap: it cannot fill
return meshconditionsclient.Observation{}, false // no module's, or with no cap: it cannot fill
}
// Compared in bytes, not rounded percent: 74.9% is not 75%.
atRaise := r.Bytes*100 >= r.MaxBytes*fillRaiseAt
aboveClear := r.Bytes*100 >= r.MaxBytes*fillClearBelow
if !atRaise && !(open && aboveClear) {
return Observation{}, false
return meshconditionsclient.Observation{}, false
}
pct := r.Bytes * 100 / r.MaxBytes
return Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: Warning,
Subject: Subject{Scope: ScopeBus, ID: stream},
return meshconditionsclient.Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeBus, ID: stream,
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
module, kind, name, mibWords(r.Bytes), mibWords(r.MaxBytes), pct),
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes,
@@ -86,7 +87,7 @@ func fillObservation(stream string, r fillRow, open bool) (Observation, bool) {
// probeFill is D-fill: every module's bucket and log on the bus judged against its cap. The bus module not answering
// is the probe failing to run, which clears nothing; a finding already written counts as open.
func probeFill(_ context.Context, c *Checker) ([]Observation, error) {
func probeFill(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
raw, err := c.Ask(StreamsTool, map[string]any{})
if err != nil {
return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err)
@@ -102,7 +103,7 @@ func probeFill(_ context.Context, c *Checker) ([]Observation, error) {
return nil, fmt.Errorf("its own findings cannot be read, so a fill between the thresholds cannot be judged: %w", err)
}
sort.Slice(a.Detail, func(i, j int) bool { return a.Detail[i].Name < a.Detail[j].Name })
var out []Observation
var out []meshconditionsclient.Observation
for _, r := range a.Detail {
if o, said := fillObservation(r.Name, r, written.has(fillKey(r.Name))); said {
out = append(out, o)
modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go
+8 -7
View File
@@ -10,9 +10,6 @@ import (
"context"
"encoding/json"
"fmt"
watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"net"
"regexp"
"slices"
@@ -20,6 +17,10 @@ import (
"strings"
"sync"
watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
)
@@ -127,7 +128,7 @@ var lookupHost = func(ctx context.Context, host string) ([]string, error) {
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are every machine's
// private address and the address its endpoint names, from placement; the ban lists are asked of each machine's holder
// of node-intrusion-prevention that is heard from.
func probeBans(ctx context.Context, c *Checker) ([]Observation, error) {
func probeBans(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
places, err := c.placements()
if err != nil {
return nil, err
@@ -182,7 +183,7 @@ func probeBans(ctx context.Context, c *Checker) ([]Observation, error) {
}(i, node)
}
wg.Wait()
var out []Observation
var out []meshconditionsclient.Observation
var unasked []string
for i, node := range holders {
if errs[i] != nil {
@@ -205,8 +206,8 @@ func probeBans(ctx context.Context, c *Checker) ([]Observation, error) {
}
}
if len(banned) > 0 {
out = append(out, Observation{Kind: "own-address-banned", Ref: "bans-the-mesh", Severity: Urgent,
Subject: Subject{Scope: ScopeMachine, ID: node, Machine: node},
out = append(out, meshconditionsclient.Observation{Kind: "own-address-banned", Ref: "bans-the-mesh", Severity: meshconditionsclient.Urgent,
Scope: ScopeMachine, ID: node, Machine: node,
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es) — %s: the mesh is locked out "+
"of itself there (ADR 0186)", node, len(banned), strings.Join(whose, ", ")),
Said: strings.Join(banned, ", "),
modules/mesh-self-check/cmd/mesh-self-check/probes_root.go
+9 -7
View File
@@ -9,13 +9,15 @@ package main
import (
"context"
"fmt"
engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"sort"
"strings"
"sync"
"time"
engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
)
// The engine's words and bounds the judgement reads (mesh-host rootsearch).
@@ -203,7 +205,7 @@ func (c *Checker) confined(node string, place placing.Placement, heard bool) (co
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest
// statement, unable to become root without a person (ADR 0266).
func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) {
func probeAgentAccounts(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
places, err := c.placements()
if err != nil {
return nil, err
@@ -217,7 +219,7 @@ func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) {
nodes = append(nodes, n)
}
sort.Strings(nodes)
var out []Observation
var out []meshconditionsclient.Observation
for _, n := range nodes {
if places[n].Decided.AgentAccount == "" {
continue
@@ -231,8 +233,8 @@ func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) {
if v.Confined || v.Quiet {
continue
}
out = append(out, Observation{Kind: "agent-can-become-root", Ref: "agent-root", Severity: Urgent,
Subject: Subject{Scope: ScopeMachine, ID: n, Machine: n},
out = append(out, meshconditionsclient.Observation{Kind: "agent-can-become-root", Ref: "agent-root", Severity: meshconditionsclient.Urgent,
Scope: ScopeMachine, ID: n, Machine: n,
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+
"answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why),
Said: v.Why,
modules/mesh-self-check/cmd/mesh-self-check/probes_test.go
+10 -8
View File
@@ -4,17 +4,19 @@ import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
assignerclient "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
placing "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
@@ -105,7 +107,7 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) {
return nil, errors.New("nats: no responders available for request")
}
func keysOf(found []Observation) []string {
func keysOf(found []meshconditionsclient.Observation) []string {
var out []string
for _, f := range found {
out = append(out, f.Key())
@@ -117,7 +119,7 @@ func keysOf(found []Observation) []string {
func TestTheBusProbes(t *testing.T) {
cases := []struct {
name string
probe func(context.Context, *Checker) ([]Observation, error)
probe func(context.Context, *Checker) ([]meshconditionsclient.Observation, error)
bus bus
want []string
err string
@@ -305,9 +307,9 @@ func placed(s *memStore) map[string]placing.Placement {
// filling is a store holding the self-check's written fill finding of a stream: the condition open.
func filling(stream string) *memStore {
s := newStore()
o := Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream},
o := meshconditionsclient.Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: stream,
Summary: "filling", Source: "D-fill"}
raw, _ := json.Marshal(o.Finding("run", time.Now()))
raw, _ := json.Marshal(o.Finding(Module, "run", time.Now()))
_ = s.Put(FindingsState, o.Key(), json.RawMessage(raw))
return s
}
modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go
+12 -11
View File
@@ -17,6 +17,7 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
meshselfcheck "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck"
)
@@ -94,7 +95,7 @@ type Probe struct {
// Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6):
// answered deferred, never a pass.
Deferred string
Run func(ctx context.Context, c *Checker) ([]Observation, error)
Run func(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error)
}
// ProbeVerdict, Counts and Run are a run as the seat mesh-self-check's contract holds it: the body of `checked`, the controller's
@@ -143,7 +144,7 @@ func (c *Checker) LastRun() *Run {
}
type result struct {
found []Observation
found []meshconditionsclient.Observation
err error
took time.Duration
}
@@ -178,9 +179,9 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
written, readErr := c.written()
seen := map[string]bool{}
want := map[string]Observation{} // every finding this run writes, by key
ran := map[string]bool{} // the sources whose findings this run judged whole
var blind []Observation
want := map[string]meshconditionsclient.Observation{} // every finding this run writes, by key
ran := map[string]bool{} // the sources whose findings this run judged whole
var blind []meshconditionsclient.Observation
for i, p := range c.Registry {
if !on(p.ID) {
// Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no
@@ -198,8 +199,8 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
case r.err != nil:
v.Verdict, v.Error = VerdictFailedToRun, r.err.Error()
run.Counts.FailedToRun++
blind = append(blind, Observation{Kind: "probe-failed", Ref: "failed", Severity: Warning, Confirm: true,
Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf,
blind = append(blind, meshconditionsclient.Observation{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeProbe, ID: p.ID, Source: sourceSelf,
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
Said: firstLine(r.err.Error())})
default:
@@ -212,7 +213,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
f = worded(f)
key := f.Key()
seen[key] = true
if f.Confirm && !c.seenBefore(key) && !written.has(key) {
if confirm(f) && !c.seenBefore(key) && !written.has(key) {
v.Unconfirmed = append(v.Unconfirmed, f.Summary)
continue
}
@@ -321,7 +322,7 @@ func (c *Checker) written() (writtenFindings, error) {
// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding
// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4).
func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error {
func (c *Checker) write(written writtenFindings, want map[string]meshconditionsclient.Observation, ran map[string]bool, run string) error {
var errs []string
keys := make([]string, 0, len(want))
for k := range want {
@@ -330,8 +331,8 @@ func (c *Checker) write(written writtenFindings, want map[string]Observation, ra
sort.Strings(keys)
now := c.Now()
for _, k := range keys {
f := want[k].Finding(run, now)
if was, ok := written[k]; ok && says(was) == says(f) {
f := want[k].Finding(Module, run, now)
if was, ok := written[k]; ok && meshconditionsclient.Says(was) == meshconditionsclient.Says(f) {
continue
}
raw, err := json.Marshal(f)
modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go
+26 -21
View File
@@ -5,13 +5,15 @@ import (
"encoding/json"
"errors"
"fmt"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// memStore is the bus's buckets in memory, counting the writes and the reads one key at a time, and handing every
@@ -113,29 +115,29 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) {
// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can
// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1).
func TestARunWritesFindingsAsTheyChange(t *testing.T) {
var found []Observation
var found []meshconditionsclient.Observation
var fail error
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }}
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { return found, fail }}
deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"}
missing := Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"},
missing := meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "EVENTS",
Summary: "the stream EVENTS is not on the bus"}
behind := Observation{Kind: "bus-reconnects", Ref: "behind", Severity: Warning, Confirm: true,
Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"}
behind := meshconditionsclient.Observation{Kind: kindDataUnmeasured, Ref: "behind", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeBus, ID: "EVENTS.controller", Summary: "far behind"}
store := newStore()
c, said := checker(store, steady, deferred)
steps := []struct {
name string
found []Observation
found []meshconditionsclient.Observation
fail error
want []string // finding keys written after the run
verdict string
missingPs int // puts of the missing finding so far
}{
{"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"seen twice: written", []Observation{missing, behind}, nil,
{"a finding is written", []meshconditionsclient.Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"the same again is no write", []meshconditionsclient.Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"one look can be wrong: held", []meshconditionsclient.Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"seen twice: written", []meshconditionsclient.Observation{missing, behind}, nil,
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"blind once: clears nothing, held", nil, errors.New("timed out"),
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1},
@@ -187,7 +189,7 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) {
t.Errorf("the last verdict counts %+v", r.Counts)
}
// A finding carries the operator's words of its kind, the probe that found it and who observed it.
found, fail = []Observation{missing}, nil
found, fail = []meshconditionsclient.Observation{missing}, nil
c.RunOnce(context.Background(), "the test")
raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing")
f, err := decodeFinding(raw)
@@ -210,9 +212,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
within := Within
Within = 50 * time.Millisecond
t.Cleanup(func() { Within = within })
for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){
"panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") },
"overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil },
for name, run := range map[string]func(context.Context, *Checker) ([]meshconditionsclient.Observation, error){
"panic": func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { panic("boom") },
"overrun": func(ctx context.Context, _ *Checker) ([]meshconditionsclient.Observation, error) {
time.Sleep(time.Second)
return nil, nil
},
} {
c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run})
c.Now = time.Now
@@ -227,12 +232,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
// every state the module keeps (ADR 0564 §12).
func TestTheVerbsAndReads(t *testing.T) {
store := newStore()
many := make([]Observation, 25)
many := make([]meshconditionsclient.Observation, 25)
for i := range many {
many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"}
many[i] = meshconditionsclient.Observation{Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: fmt.Sprintf("S%02d", i), Summary: "x"}
}
c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong",
Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }})
Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { return many, nil }})
if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") {
t.Fatalf("before a run: %v", err)
}
@@ -278,11 +283,11 @@ func TestTheVerbsAndReads(t *testing.T) {
// and is reported by the verb as off by setting; the default list is D13 and D-fill (the operator, 2026-10-30).
func TestOnlyTheProbesTheSettingListsRun(t *testing.T) {
ran := map[string]int{}
finds := Observation{Ref: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "gone"}
finds := meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "EVENTS", Summary: "gone"}
probe := func(id string) Probe {
return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) {
return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) {
ran[id]++
return []Observation{finds}, nil
return []meshconditionsclient.Observation{finds}, nil
}}
}
store := newStore()
modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go
+2 -1
View File
@@ -26,6 +26,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
@@ -295,7 +296,7 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) {
t.Fatalf("the missing stream is not a finding on the bus: %v", err)
}
f, err := decodeFinding(e.Value())
if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
if err != nil || f.Kind != "stream-wrong" || f.Severity != meshconditionsclient.Urgent || f.Source != "D7" {
t.Errorf("the finding is %s", e.Value())
}
verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState)
modules/mesh-self-check/cmd/mesh-self-check/words.go
+62
View File
@@ -0,0 +1,62 @@
package main
// The operator's words for the kinds the self-check raises; the observation and its finding are the conditions
// client's (novox/hq ADR 0564 §2, ADR 0634 §4).
import "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
// worded gives a finding the operator's words registered for its kind, where it says none of its own.
func worded(f meshconditionsclient.Observation) meshconditionsclient.Observation {
w, ok := words[f.Kind]
if !ok {
return f
}
if f.Headline == "" {
f.Headline = w.Headline
}
if f.Explanation == "" {
f.Explanation = w.Explanation
}
if f.Resolved == "" {
f.Resolved = w.Resolved
}
if f.Needs == "" {
f.Needs = w.Needs
}
return f
}
type plain struct{ Headline, Explanation, Resolved, Needs string }
// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the
// controller's name for itself replaced by the self-check's where the words spoke of it.
var words = map[string]plain{
"stream-wrong": {Headline: "Part of the bus's storage is wrong",
Needs: "check the machine the bus runs on; the details say what is missing.",
Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
Resolved: "The bus's storage is right again"},
"bus-reconnects": {Headline: "A client keeps losing the bus",
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " +
"it reconnects, what it says and what it is asked waits.",
Resolved: "Resolved: the client stays connected"},
"core-unhealthy": {Headline: "The bus is not healthy",
Needs: "check the machine the bus runs on; the details say what fails.",
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " +
"calls may fail until it recovers.",
Resolved: "The bus is healthy again"},
"watchdogs-silent": {Headline: "The mesh's watchdogs stopped",
Needs: "restart the watchman if this stays, from the mesh MCP server.",
Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " +
"signals are not noticed.",
Resolved: "The watchdogs run again"},
"probe-failed": {Headline: "A check of the mesh could not run",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " +
"every few minutes.",
Resolved: "The check runs again"},
}
// confirm says one look can be wrong about the finding: it is written only when two runs in a row see it (issue 277).
// The kinds are the self-check's own: a probe that could not run, and a holder that measured nothing of the data.
func confirm(o meshconditionsclient.Observation) bool {
return o.Kind == "probe-failed" || o.Kind == kindDataUnmeasured
}