mesh-self-check: a run is the seat's checked event and a finding the seat mesh-conditions' value (B5d; ADR 0634 §1, §4; issue 1966)

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
2026-11-05 15:09:00 +01:00
parent 0ae418b43c
commit 97cc9b1ccf
8 changed files with 58 additions and 51 deletions
@@ -2,35 +2,36 @@ package main
// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an // What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an
// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong // Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong
// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key // about it. It is written into the bucket `findings` as the seat mesh-conditions' finding, whose key is the controller's key
// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. // for the same condition, so the condition the keeper raises from it and the one the controller raised share one key.
import ( import (
"encoding/json" "encoding/json"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
) )
// The severities and scopes, as mesh-sdk's health names them. // The severities and scopes, as mesh-sdk's health names them.
const ( const (
Urgent = health.Urgent Urgent = meshconditionsclient.Urgent
Warning = health.Warning Warning = meshconditionsclient.Warning
ScopeBus = health.ScopeBus ScopeBus = meshconditionsclient.ScopeBus
ScopeCore = health.ScopeCore ScopeCore = meshconditionsclient.ScopeCore
ScopeProbe = health.ScopeProbe ScopeProbe = meshconditionsclient.ScopeProbe
ScopeMesh = health.ScopeMesh ScopeMesh = meshconditionsclient.ScopeMesh
) )
// Subject is what a finding is about. // Subject is what a finding is about.
type Subject = health.Subject type Subject = meshconditions.Subject
// Observation is one thing a probe sees wrong now. // Observation is one thing a probe sees wrong now.
type Observation struct { type Observation struct {
Kind string Kind string
Ref string Ref string
Subject Subject Subject Subject
Severity health.Severity Severity string
Summary string Summary string
Said string Said string
Headline string Headline string
@@ -49,14 +50,14 @@ func (o Observation) Key() string {
if token == "" { if token == "" {
token = o.Kind token = o.Kind
} }
return health.Key(o.Subject.Scope, o.Subject.ID, token) return meshconditionsclient.Key(o.Subject.Scope, o.Subject.ID, token)
} }
// Finding is the observation as written by run at, its evidence the one line it said. // Finding is the observation as written by run at, its evidence the one line it said.
func (o Observation) Finding(run string, at time.Time) health.Finding { func (o Observation) Finding(run string, at time.Time) meshconditions.FindingsByKeyValue {
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} Observer: Module, Run: run, At: at.UTC(), Evidence: []meshconditions.Evidence{}}
if o.Ref != "" && o.Ref != o.Kind { if o.Ref != "" && o.Ref != o.Kind {
f.Ref = o.Ref f.Ref = o.Ref
} }
@@ -64,17 +65,17 @@ func (o Observation) Finding(run string, at time.Time) health.Finding {
if said == "" { if said == "" {
said = o.Summary said = o.Summary
} }
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) f.Evidence = append(f.Evidence, meshconditions.Evidence{At: at.UTC(), Said: said})
return f return f
} }
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write // says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
// (ADR 0289 §1). // (ADR 0289 §1).
func says(f health.Finding) string { func says(f meshconditions.FindingsByKeyValue) string {
f.At, f.Run, f.V = time.Time{}, "", 0 f.At, f.Run, f.V = time.Time{}, "", 0
ev := make([]health.Evidence, len(f.Evidence)) ev := make([]meshconditions.Evidence, len(f.Evidence))
for i, e := range f.Evidence { for i, e := range f.Evidence {
ev[i] = health.Evidence{Said: e.Said} ev[i] = meshconditions.Evidence{Said: e.Said}
} }
f.Evidence = ev f.Evidence = ev
raw, _ := json.Marshal(f) raw, _ := json.Marshal(f)
@@ -23,7 +23,6 @@ import (
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood" "git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
) )
// The states D13 reads (declared under `reads`). // The states D13 reads (declared under `reads`).
@@ -201,7 +200,7 @@ func probeData(ctx context.Context, c *Checker) ([]Observation, error) {
return dataFindings(f), nil return dataFindings(f), nil
} }
func severityOf(class string) health.Severity { func severityOf(class string) string {
if class == catalogue.ClassIrreplaceable { if class == catalogue.ClassIrreplaceable {
return Urgent return Urgent
} }
@@ -264,7 +263,7 @@ func firstSeen(r diskreadings.Reading, now time.Time) time.Time {
return now return now
} }
func obs(kind string, id, machine string, sev health.Severity, summary, said string) Observation { func obs(kind string, id, machine string, sev string, summary, said string) Observation {
return Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said, return Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said,
Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}} Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}}
} }
@@ -17,8 +17,8 @@ import (
"strings" "strings"
"sync" "sync"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
) )
// The states the machine probes read (declared under `reads`), and the verb D8 asks each node. // The states the machine probes read (declared under `reads`), and the verb D8 asks each node.
@@ -28,7 +28,7 @@ const (
PresenceRead = delivery.WatchmanModule + ".presence" PresenceRead = delivery.WatchmanModule + ".presence"
BannedSeat = "node-intrusion-prevention" BannedSeat = "node-intrusion-prevention"
BannedVerb = "seat:" + BannedSeat + ".banned" BannedVerb = "seat:" + BannedSeat + ".banned"
ScopeMachine = health.ScopeMachine ScopeMachine = meshconditionsclient.ScopeMachine
) )
// placements is every node's placement, by node, from the state's keys in either form: a node key and its module keys, // placements is every node's placement, by node, from the state's keys in either form: a node key and its module keys,
@@ -14,7 +14,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
) )
// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed. // bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed.
@@ -302,7 +301,7 @@ func filling(stream string) *memStore {
s := newStore() s := newStore()
o := Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream}, o := Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream},
Summary: "filling", Source: "D-fill"} Summary: "filling", Source: "D-fill"}
raw, _ := health.EncodeFinding(o.Finding("run", time.Now())) raw, _ := json.Marshal(o.Finding("run", time.Now()))
_ = s.Put(FindingsState, o.Key(), json.RawMessage(raw)) _ = s.Put(FindingsState, o.Key(), json.RawMessage(raw))
return s return s
} }
@@ -17,7 +17,8 @@ import (
"sync" "sync"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/health" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
meshselfcheck "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck"
) )
// Module and Seat are this module and the seat it holds once for the mesh. // Module and Seat are this module and the seat it holds once for the mesh.
@@ -28,11 +29,11 @@ const (
// The states it keeps, and the event that ends every run. // The states it keeps, and the event that ends every run.
const ( const (
FindingsState = health.FindingsState FindingsState = meshconditions.StateFindings
VerdictState = "verdict" VerdictState = "verdict"
VerdictsState = "verdicts" VerdictsState = "verdicts"
LastKey = "last" LastKey = "last"
CheckedEvent = health.EventChecked CheckedEvent = meshselfcheck.EventChecked
// CheckedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its // CheckedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
// consumers hear (catalogue.SeatSays). // consumers hear (catalogue.SeatSays).
CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent
@@ -45,12 +46,12 @@ var (
Within = 30 * time.Second Within = 30 * time.Second
) )
// The verdicts a probe can have, as mesh-sdk's health names them. // The verdicts a probe can have, as the seat's contract names them.
const ( const (
VerdictPass = health.VerdictPassed VerdictPass = meshselfcheck.ProbeVerdictVerdictPassed
VerdictFail = health.VerdictFailed VerdictFail = meshselfcheck.ProbeVerdictVerdictFailed
VerdictFailedToRun = health.VerdictFailedToRun VerdictFailedToRun = meshselfcheck.ProbeVerdictVerdictFailedToRun
VerdictDeferred = health.VerdictDeferred VerdictDeferred = meshselfcheck.ProbeVerdictVerdictDeferred
) )
// sourceSelf is what writes a probe's failure to run: the self-check itself. // sourceSelf is what writes a probe's failure to run: the self-check itself.
@@ -96,12 +97,12 @@ type Probe struct {
Run func(ctx context.Context, c *Checker) ([]Observation, error) Run func(ctx context.Context, c *Checker) ([]Observation, error)
} }
// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's // ProbeVerdict, Counts and Run are a run as the seat mesh-self-check's contract holds it: the body of `checked`, the controller's
// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it. // doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it.
type ( type (
ProbeVerdict = health.ProbeVerdict ProbeVerdict = meshselfcheck.ProbeVerdict
Counts = health.CheckedCounts Counts = meshselfcheck.CheckedEventCounts
Run = health.Checked Run = meshselfcheck.CheckedEvent
) )
// Checker is the self-check: the registry and what its probes reach. // Checker is the self-check: the registry and what its probes reach.
@@ -157,8 +158,8 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
n := c.n n := c.n
c.mu.Unlock() c.mu.Unlock()
started := c.Now() started := c.Now()
run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(), run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: &started, Counts: &Counts{},
IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why} IntervalSeconds: int64(Every / time.Second), Controller: c.Host, Why: why}
on := c.listed() on := c.listed()
results := make([]result, len(c.Registry)) results := make([]result, len(c.Registry))
@@ -253,7 +254,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run {
run.At = c.Now().UTC() run.At = c.Now().UTC()
run.Took = run.At.Sub(started).Round(time.Millisecond).String() run.Took = run.At.Sub(started).Round(time.Millisecond).String()
c.mu.Lock() c.mu.Lock()
run.Unsaid = c.unsaid run.Unsaid = int64(c.unsaid)
c.mu.Unlock() c.mu.Unlock()
c.keep(run) c.keep(run)
return run return run
@@ -290,7 +291,7 @@ func (c *Checker) seenBefore(key string) bool {
} }
// writtenFindings are the findings in the bucket, by key. // writtenFindings are the findings in the bucket, by key.
type writtenFindings map[string]health.Finding type writtenFindings map[string]meshconditions.FindingsByKeyValue
func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok }
@@ -309,8 +310,8 @@ func (c *Checker) written() (writtenFindings, error) {
if !found { if !found {
continue continue
} }
f, err := health.DecodeFinding(raw) var f meshconditions.FindingsByKeyValue
if err != nil { if err := json.Unmarshal(raw, &f); err != nil {
return out, fmt.Errorf("the finding %s is not readable: %w", k, err) return out, fmt.Errorf("the finding %s is not readable: %w", k, err)
} }
out[k] = f out[k] = f
@@ -333,7 +334,7 @@ func (c *Checker) write(written writtenFindings, want map[string]Observation, ra
if was, ok := written[k]; ok && says(was) == says(f) { if was, ok := written[k]; ok && says(was) == says(f) {
continue continue
} }
raw, err := health.EncodeFinding(f) raw, err := json.Marshal(f)
if err == nil { if err == nil {
err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) err = c.Store.Put(FindingsState, k, json.RawMessage(raw))
} }
@@ -388,7 +389,7 @@ func Overview(run Run) Run {
for i, p := range run.Probes { for i, p := range run.Probes {
total := len(p.Found) + len(p.Unconfirmed) total := len(p.Found) + len(p.Unconfirmed)
if total > overviewFindings { if total > overviewFindings {
p.FoundCount = len(p.Found) p.FoundCount = int64(len(p.Found))
if len(p.Found) > overviewFindings { if len(p.Found) > overviewFindings {
p.Found = p.Found[:overviewFindings:overviewFindings] p.Found = p.Found[:overviewFindings:overviewFindings]
} }
@@ -396,7 +397,7 @@ func Overview(run Run) Run {
p.Unconfirmed = p.Unconfirmed[:room:room] p.Unconfirmed = p.Unconfirmed[:room:room]
} }
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all", p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all",
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID) p.FoundCount, int64(total)-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID)
} }
probes[i] = p probes[i] = p
} }
@@ -5,14 +5,13 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"slices" "slices"
"sort" "sort"
"strings" "strings"
"sync" "sync"
"testing" "testing"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/health"
) )
// memStore is the bus's buckets in memory, counting the writes and the reads one key at a time, and handing every // memStore is the bus's buckets in memory, counting the writes and the reads one key at a time, and handing every
@@ -191,7 +190,7 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) {
found, fail = []Observation{missing}, nil found, fail = []Observation{missing}, nil
c.RunOnce(context.Background(), "the test") c.RunOnce(context.Background(), "the test")
raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing")
f, err := health.DecodeFinding(raw) f, err := decodeFinding(raw)
if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" ||
f.Explanation == "" || f.Ref != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { f.Explanation == "" || f.Ref != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" {
t.Errorf("the written finding is %+v", f) t.Errorf("the written finding is %+v", f)
@@ -319,3 +318,10 @@ func TestOnlyTheProbesTheSettingListsRun(t *testing.T) {
t.Fatal("no setting runs every probe, and the default list is D13 and D-fill") t.Fatal("no setting runs every probe, and the default list is D13 and D-fill")
} }
} }
// decodeFinding reads a finding as the seat mesh-conditions lays it out.
func decodeFinding(raw []byte) (meshconditions.FindingsByKeyValue, error) {
var f meshconditions.FindingsByKeyValue
err := json.Unmarshal(raw, &f)
return f, err
}
@@ -26,7 +26,6 @@ import (
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus" "git.novox.be/novox/mesh-sdk/go/testbus"
) )
@@ -295,7 +294,7 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("the missing stream is not a finding on the bus: %v", err) t.Fatalf("the missing stream is not a finding on the bus: %v", err)
} }
f, err := health.DecodeFinding(e.Value()) f, err := decodeFinding(e.Value())
if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
t.Errorf("the finding is %s", e.Value()) t.Errorf("the finding is %s", e.Value())
} }
+2
View File
@@ -31,3 +31,5 @@ replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../dis
require git.novox.be/novox/mesh-catalog/modules/restic/backupgood v0.0.0-00010101000000-000000000000 require git.novox.be/novox/mesh-catalog/modules/restic/backupgood v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/restic/backupgood => ../restic/backupgood replace git.novox.be/novox/mesh-catalog/modules/restic/backupgood => ../restic/backupgood
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck v0.0.0-00010101000000-000000000000