Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)
mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…
mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…
One commit with exactly the tree of the reviewed head35b4509c0a, on main12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of
|
||||
// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds
|
||||
// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts`
|
||||
// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is
|
||||
// the MCP channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// runtimeStore is the bus, through the runtime that launched this bundle.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Put(state, key string, value any) error {
|
||||
_, err := stdio.State(state).Put(key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// NewChecker is the self-check over the runtime.
|
||||
func NewChecker() *Checker {
|
||||
return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"),
|
||||
Enabled: enabledProbes,
|
||||
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
|
||||
Publish: stdio.SeatPublish}
|
||||
}
|
||||
|
||||
func main() {
|
||||
c := NewChecker()
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
c.Keep(context.Background())
|
||||
}()
|
||||
if err := stdio.Serve("", Tools(c)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// enabledProbes is the setting `probes`, read at each run so a change applies at the next; with it unreadable, the
|
||||
// default list.
|
||||
func enabledProbes() []string {
|
||||
raw, err := stdio.Config("probes")
|
||||
if err != nil {
|
||||
logf("the setting probes cannot be read (%v); the default list runs", err)
|
||||
return DefaultProbes
|
||||
}
|
||||
var list []string
|
||||
if json.Unmarshal(raw, &list) != nil {
|
||||
logf("the setting probes is not a list of probe ids (%s); the default list runs", raw)
|
||||
return DefaultProbes
|
||||
}
|
||||
return list
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
RunsAs string `json:"runs-as"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Emits []string `json:"emits"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
State []json.RawMessage `json:"state"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T) manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves
|
||||
// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts,
|
||||
// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by
|
||||
// itself a tool that starts it by hand (ADR 0564 §12).
|
||||
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if m.Module != Module || m.RunsAs != Module {
|
||||
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
|
||||
t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs)
|
||||
}
|
||||
// checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims.
|
||||
if len(m.Emits) != 0 {
|
||||
t.Errorf("emits %v", m.Emits)
|
||||
}
|
||||
if !slices.Equal(m.Invokes, Invokes) {
|
||||
t.Errorf("invokes %v, want what the probes ask, %v", m.Invokes, Invokes)
|
||||
}
|
||||
if !slices.Equal(m.Reads, Reads) {
|
||||
t.Errorf("reads %v, want what the probes read, %v", m.Reads, Reads)
|
||||
}
|
||||
states := map[string]int{}
|
||||
for _, s := range m.State {
|
||||
var d struct {
|
||||
Name string `json:"name"`
|
||||
TTL int `json:"ttl-seconds"`
|
||||
}
|
||||
if json.Unmarshal(s, &d.Name) != nil {
|
||||
_ = json.Unmarshal(s, &d)
|
||||
}
|
||||
states[d.Name] = d.TTL
|
||||
}
|
||||
if states[VerdictsState] != 7*24*60*60 {
|
||||
t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState])
|
||||
}
|
||||
// ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it.
|
||||
readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool,
|
||||
"the run": DoctorTool + " run=true"}
|
||||
tools := map[string]bool{}
|
||||
for _, tool := range Tools(&Checker{}) {
|
||||
tools[tool.Name] = true
|
||||
}
|
||||
for state := range states {
|
||||
if _, ok := readBy[state]; !ok {
|
||||
t.Errorf("the state %s has no tool that reads it", state)
|
||||
}
|
||||
}
|
||||
for what, tool := range readBy {
|
||||
name, _, _ := cutSpace(tool)
|
||||
if !tools[name] {
|
||||
t.Errorf("%s is read or started by %s, which is not served", what, tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cutSpace(s string) (string, string, bool) {
|
||||
for i, r := range s {
|
||||
if r == ' ' {
|
||||
return s[:i], s[i+1:], true
|
||||
}
|
||||
}
|
||||
return s, "", false
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an
|
||||
// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong
|
||||
// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key
|
||||
// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The severities and scopes, as mesh-sdk's health names them.
|
||||
const (
|
||||
Urgent = health.Urgent
|
||||
Warning = health.Warning
|
||||
ScopeBus = health.ScopeBus
|
||||
ScopeCore = health.ScopeCore
|
||||
ScopeProbe = health.ScopeProbe
|
||||
ScopeMesh = health.ScopeMesh
|
||||
)
|
||||
|
||||
// Subject is what a finding is about.
|
||||
type Subject = health.Subject
|
||||
|
||||
// Observation is one thing a probe sees wrong now.
|
||||
type Observation struct {
|
||||
Kind string
|
||||
Token string
|
||||
Subject Subject
|
||||
Severity health.Severity
|
||||
Summary string
|
||||
Said string
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
Needs string
|
||||
// Source is the probe that saw it, or the self-check itself for a probe that could not run.
|
||||
Source string
|
||||
// Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277).
|
||||
Confirm bool
|
||||
}
|
||||
|
||||
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = o.Kind
|
||||
}
|
||||
return health.Key(o.Subject.Scope, o.Subject.ID, token)
|
||||
}
|
||||
|
||||
// Finding is the observation as written by run at, its evidence the one line it said.
|
||||
func (o Observation) Finding(run string, at time.Time) health.Finding {
|
||||
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
|
||||
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
|
||||
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}}
|
||||
if o.Token != "" && o.Token != o.Kind {
|
||||
f.Token = o.Token
|
||||
}
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said})
|
||||
return f
|
||||
}
|
||||
|
||||
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
|
||||
// (ADR 0289 §1).
|
||||
func says(f health.Finding) string {
|
||||
f.At, f.Run, f.V = time.Time{}, "", 0
|
||||
ev := make([]health.Evidence, len(f.Evidence))
|
||||
for i, e := range f.Evidence {
|
||||
ev[i] = health.Evidence{Said: e.Said}
|
||||
}
|
||||
f.Evidence = ev
|
||||
raw, _ := json.Marshal(f)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// worded gives a finding the operator's words registered for its kind, where it says none of its own.
|
||||
func worded(f Observation) Observation {
|
||||
w, ok := words[f.Kind]
|
||||
if !ok {
|
||||
return f
|
||||
}
|
||||
if f.Headline == "" {
|
||||
f.Headline = w.Headline
|
||||
}
|
||||
if f.Explanation == "" {
|
||||
f.Explanation = w.Explanation
|
||||
}
|
||||
if f.Resolved == "" {
|
||||
f.Resolved = w.Resolved
|
||||
}
|
||||
if f.Needs == "" {
|
||||
f.Needs = w.Needs
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
type plain struct{ Headline, Explanation, Resolved, Needs string }
|
||||
|
||||
// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the
|
||||
// controller's name for itself replaced by the self-check's where the words spoke of it.
|
||||
var words = map[string]plain{
|
||||
"consumer-lost": {Headline: "A listener on the bus is gone",
|
||||
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
||||
Resolved: "The listener is back"},
|
||||
"consumer-wrong": {Headline: "A listener on the bus is set up wrong",
|
||||
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.",
|
||||
Resolved: "The listener is set up right again"},
|
||||
"consumer-behind": {Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " +
|
||||
"restarts it if it stays.",
|
||||
Resolved: "The listener caught up"},
|
||||
"stream-wrong": {Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
||||
Resolved: "The bus's storage is right again"},
|
||||
"bus-reconnects": {Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " +
|
||||
"it reconnects, what it says and what it is asked waits.",
|
||||
Resolved: "Resolved: the client stays connected"},
|
||||
"core-unhealthy": {Headline: "The bus is not healthy",
|
||||
Needs: "check the machine the bus runs on; the details say what fails.",
|
||||
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " +
|
||||
"calls may fail until it recovers.",
|
||||
Resolved: "The bus is healthy again"},
|
||||
"watchdogs-silent": {Headline: "The mesh's watchdogs stopped",
|
||||
Needs: "restart the watchman if this stays, from the mesh MCP server.",
|
||||
Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " +
|
||||
"signals are not noticed.",
|
||||
Resolved: "The watchdogs run again"},
|
||||
"probe-failed": {Headline: "A check of the mesh could not run",
|
||||
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " +
|
||||
"every few minutes.",
|
||||
Resolved: "The check runs again"},
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
package main
|
||||
|
||||
// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller,
|
||||
// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home
|
||||
// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the
|
||||
// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the
|
||||
// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and
|
||||
// the bus step (its home is issue 1953's).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat.
|
||||
const (
|
||||
StreamsTool = "nats.nats_streams"
|
||||
ClosedTool = "nats.nats_closed_connections"
|
||||
SignalsVerb = "seat:mesh-watchdogs.signals"
|
||||
notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; "
|
||||
storeOnly = "held only in the controller's store today"
|
||||
behindBound = 1000
|
||||
reconnectMax = 12
|
||||
)
|
||||
|
||||
// Invokes are the tools and seat verbs the probes ask, and Reads the states they read: the manifest's `invokes` and
|
||||
// `reads`, which its test holds to these.
|
||||
var (
|
||||
Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb}
|
||||
Reads = []string{PlacementRead, SeatHoldersRead, PresenceRead, HealthRead, ReadingsRead, GoodRead, ModulesRead, ManifestsRead}
|
||||
)
|
||||
|
||||
// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe
|
||||
// that moves here is a row added, and the registry test holds it to ADR 0564's table.
|
||||
func Registry() []Probe {
|
||||
return []Probe{
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong",
|
||||
Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " +
|
||||
"is heard from", From: "issues 208, 218", Kind: "holder-silent",
|
||||
Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " +
|
||||
"records are " + storeOnly + ", and the watchman's presence has no read granted to this module"},
|
||||
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld",
|
||||
Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly},
|
||||
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong",
|
||||
Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers},
|
||||
{ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208",
|
||||
Kind: "stream-wrong", Run: probeStreams},
|
||||
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||
From: "issue 238", Kind: "own-address-banned", Run: probeBans},
|
||||
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " +
|
||||
"cleanup", From: "ADR 0230", Kind: "cleanup-waiting",
|
||||
Deferred: "the retirement records of providers' consumers: " + storeOnly},
|
||||
{ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " +
|
||||
"by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved",
|
||||
Deferred: "the bindings of provisions to their consumers: " + storeOnly},
|
||||
{ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " +
|
||||
"backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank",
|
||||
Raises: []string{kindDataMissing, kindDataQuiet, kindBackupStale, kindArrayDegraded, kindProtectionMissing,
|
||||
kindDataUnmeasured},
|
||||
Run: probeData},
|
||||
{ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " +
|
||||
"bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects},
|
||||
{ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " +
|
||||
"statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
|
||||
Kind: "agent-can-become-root", Run: probeAgentAccounts},
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free",
|
||||
Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly},
|
||||
{ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " +
|
||||
"filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: "bucket-or-log-filling", Run: probeFill},
|
||||
{ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it",
|
||||
From: "ADR 0376 §2", Kind: "grant-composes-nothing",
|
||||
Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly},
|
||||
{ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs},
|
||||
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " +
|
||||
"to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth},
|
||||
}
|
||||
}
|
||||
|
||||
// --- the bus's own state, as its module answers it ---------------------------------------------------------------
|
||||
|
||||
// streamsAnswer is what nats_streams answers.
|
||||
type streamsAnswer struct {
|
||||
Detail []streamRow `json:"detail"`
|
||||
}
|
||||
|
||||
type streamRow struct {
|
||||
Name string `json:"name"`
|
||||
Subjects []string `json:"subjects"`
|
||||
Retention string `json:"retention"`
|
||||
MaxPerSubject int64 `json:"max_per_subject"`
|
||||
Consumers []consumerRow `json:"consumer_detail"`
|
||||
}
|
||||
|
||||
type consumerRow struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Filters []string `json:"filters"`
|
||||
Pending int64 `json:"pending"`
|
||||
Unacked int64 `json:"unacknowledged"`
|
||||
MaxDeliver int64 `json:"max_deliver"`
|
||||
}
|
||||
|
||||
// errNoStream is a stream the bus module says it does not hold.
|
||||
var errNoStream = errors.New("no such stream")
|
||||
|
||||
// streams asks the bus module for every stream, or for one with its consumers.
|
||||
func (c *Checker) streams(stream string) ([]streamRow, error) {
|
||||
args := map[string]any{}
|
||||
if stream != "" {
|
||||
args["stream"] = stream
|
||||
}
|
||||
raw, err := c.Ask(StreamsTool, args)
|
||||
if err != nil {
|
||||
if stream != "" && strings.Contains(err.Error(), "no stream called") {
|
||||
return nil, errNoStream
|
||||
}
|
||||
return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err)
|
||||
}
|
||||
var a streamsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err)
|
||||
}
|
||||
return a.Detail, nil
|
||||
}
|
||||
|
||||
// consumerWords is a consumer as the mesh says it, with its name.
|
||||
func consumerWords(c buscompose.Consumer) string {
|
||||
return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream)
|
||||
}
|
||||
|
||||
// expectedConsumers are the durable consumers the mesh defines, by stream.
|
||||
func expectedConsumers() map[string][]buscompose.Consumer {
|
||||
out := map[string][]buscompose.Consumer{}
|
||||
for _, c := range buscompose.MeshConsumers() {
|
||||
out[c.Stream] = append(out[c.Stream], c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head.
|
||||
func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
var out []Observation
|
||||
byStream := expectedConsumers()
|
||||
streams := make([]string, 0, len(byStream))
|
||||
for s := range byStream {
|
||||
streams = append(streams, s)
|
||||
}
|
||||
sort.Strings(streams)
|
||||
for _, stream := range streams {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
rows, err := c.streams(stream)
|
||||
held := map[string]consumerRow{}
|
||||
switch {
|
||||
case errors.Is(err, errNoStream):
|
||||
case err != nil:
|
||||
return nil, err
|
||||
default:
|
||||
for _, r := range rows {
|
||||
for _, cr := range r.Consumers {
|
||||
held[cr.Name] = cr
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, want := range byStream[stream] {
|
||||
who := want.Stream + "." + want.Name
|
||||
have, ok := held[want.Name]
|
||||
if !ok {
|
||||
out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)),
|
||||
Said: "no consumer " + want.Name + " on " + want.Stream})
|
||||
continue
|
||||
}
|
||||
if differs := consumerDiffers(want, have); differs != "" {
|
||||
out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs),
|
||||
Said: differs})
|
||||
}
|
||||
if want.Stream != "NODES" && have.Pending > behindBound {
|
||||
// One look at a consumer a burst put behind is not one that stays behind: held for the next run,
|
||||
// so healer H4 resets nothing on a single sample (novox/hq issue 277).
|
||||
out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending),
|
||||
Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its
|
||||
// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is.
|
||||
func consumerDiffers(want buscompose.Consumer, have consumerRow) string {
|
||||
var differs []string
|
||||
haveFilters := slices.Clone(have.Filters)
|
||||
wantFilters := slices.Clone(want.Filters)
|
||||
sort.Strings(haveFilters)
|
||||
sort.Strings(wantFilters)
|
||||
if !slices.Equal(haveFilters, wantFilters) {
|
||||
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||
}
|
||||
if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) {
|
||||
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver))
|
||||
}
|
||||
if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush {
|
||||
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// probeStreams is D7: every stream the mesh defines exists as defined.
|
||||
func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
rows, err := c.streams("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held := map[string]streamRow{}
|
||||
for _, r := range rows {
|
||||
held[r.Name] = r
|
||||
}
|
||||
var out []Observation
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
have, ok := held[s.Name]
|
||||
if !ok {
|
||||
out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||
Said: "no stream " + s.Name})
|
||||
continue
|
||||
}
|
||||
if differs := streamDiffers(s, have); differs != "" {
|
||||
out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs),
|
||||
Said: differs})
|
||||
}
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its
|
||||
// subjects, its retention and how many it keeps per subject. Empty when nothing is.
|
||||
func streamDiffers(want buscompose.Stream, have streamRow) string {
|
||||
var differs []string
|
||||
haveSubjects := slices.Clone(have.Subjects)
|
||||
wantSubjects := slices.Clone(want.Subjects)
|
||||
sort.Strings(haveSubjects)
|
||||
sort.Strings(wantSubjects)
|
||||
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||
}
|
||||
retention, perSubject := "limits", int64(want.MaxMsgsPerSubject)
|
||||
switch want.Retention {
|
||||
case buscompose.RetentionWorkQueue:
|
||||
retention = "workqueue"
|
||||
case buscompose.RetentionLastPerSubject:
|
||||
perSubject = 1
|
||||
}
|
||||
if have.Retention != "" && have.Retention != retention {
|
||||
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||
}
|
||||
if perSubject != 0 && have.MaxPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject))
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// --- D15 -----------------------------------------------------------------------------------------------------
|
||||
|
||||
// closedAnswer is what nats_closed_connections answers.
|
||||
type closedAnswer struct {
|
||||
Hours float64 `json:"hours"`
|
||||
Reaches bool `json:"reaches"`
|
||||
Users []struct {
|
||||
User string `json:"user"`
|
||||
Closed int `json:"closed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Names []struct {
|
||||
Name string `json:"name"`
|
||||
Dropped int `json:"dropped"`
|
||||
Reasons map[string]int `json:"reasons"`
|
||||
} `json:"names"`
|
||||
} `json:"users"`
|
||||
}
|
||||
|
||||
// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour.
|
||||
func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err)
|
||||
}
|
||||
var read closedAnswer
|
||||
if err := json.Unmarshal(raw, &read); err != nil {
|
||||
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err)
|
||||
}
|
||||
hours := read.Hours
|
||||
if hours <= 0 {
|
||||
hours = 1
|
||||
}
|
||||
var out []Observation
|
||||
for _, u := range read.Users {
|
||||
if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax {
|
||||
continue // refused before it logged in is nobody's reconnect loop
|
||||
}
|
||||
reasons := map[string]int{}
|
||||
var names []string
|
||||
for _, n := range u.Names {
|
||||
if n.Dropped == 0 {
|
||||
continue
|
||||
}
|
||||
names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped))
|
||||
for r, k := range n.Reasons {
|
||||
if r != "Client Closed" {
|
||||
reasons[r] += k
|
||||
}
|
||||
}
|
||||
}
|
||||
var why []string
|
||||
for r, k := range reasons {
|
||||
why = append(why, fmt.Sprintf("%s ×%d", r, k))
|
||||
}
|
||||
sort.Strings(why)
|
||||
partial := ""
|
||||
if !read.Reaches {
|
||||
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
|
||||
}
|
||||
who, machine := busUserWords(u.User)
|
||||
out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine},
|
||||
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+
|
||||
"reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax),
|
||||
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed,
|
||||
hours, strings.Join(names, ", "), strings.Join(why, ", ")),
|
||||
Headline: capital(who) + " keeps losing the bus",
|
||||
Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+
|
||||
"again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)),
|
||||
Resolved: "Resolved: " + who + " stays connected"})
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// busUserWords is a bus user as the operator says it, and the machine it is on.
|
||||
func busUserWords(user string) (string, string) {
|
||||
switch {
|
||||
case user == "controller":
|
||||
return "the controller", ""
|
||||
case strings.HasPrefix(user, "node."):
|
||||
m := strings.TrimPrefix(user, "node.")
|
||||
return "the node-engine on " + m, m
|
||||
}
|
||||
if m, module, ok := strings.Cut(user, "."); ok {
|
||||
if module == "node-tools" {
|
||||
return "the tool runner on " + m, m
|
||||
}
|
||||
return module + " on " + m, m
|
||||
}
|
||||
return "a client of the bus", ""
|
||||
}
|
||||
|
||||
// --- DW and H-bus --------------------------------------------------------------------------------------------
|
||||
|
||||
// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the
|
||||
// watchman's pulse on `checked`, is the other half.
|
||||
func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil {
|
||||
return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"},
|
||||
Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " +
|
||||
"late signal is being said",
|
||||
Said: firstLine(err.Error())}}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it
|
||||
// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip,
|
||||
// and no answer is no pass.
|
||||
func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
rows, err := c.streams("")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, r := range rows {
|
||||
held[r.Name] = true
|
||||
}
|
||||
var problems []string
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
if !held[s.Name] {
|
||||
problems = append(problems, "the stream "+s.Name+" is missing")
|
||||
}
|
||||
}
|
||||
byStream := expectedConsumers()
|
||||
streams := make([]string, 0, len(byStream))
|
||||
for s := range byStream {
|
||||
streams = append(streams, s)
|
||||
}
|
||||
sort.Strings(streams)
|
||||
for _, stream := range streams {
|
||||
if !held[stream] {
|
||||
continue // said above
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
detail, err := c.streams(stream)
|
||||
if err != nil && !errors.Is(err, errNoStream) {
|
||||
return nil, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, r := range detail {
|
||||
for _, cr := range r.Consumers {
|
||||
have[cr.Name] = true
|
||||
}
|
||||
}
|
||||
for _, want := range byStream[stream] {
|
||||
if !have[want.Name] {
|
||||
problems = append(problems, consumerWords(want)+" is missing")
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"},
|
||||
Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil
|
||||
}
|
||||
|
||||
// sorted is findings in a stated order, so two runs over one mesh say the same.
|
||||
func sorted(out []Observation) []Observation {
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||
return out
|
||||
}
|
||||
|
||||
func capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
package main
|
||||
|
||||
// D13, the data the modules declare (novox/hq ADR 0233, ADR 0571 clause 3, ADR 0564 §6): every item of data a machine
|
||||
// declares is measured, is there, holds what it held, and is backed up within its bound. The controller composed what
|
||||
// every machine declares and asked each machine's backup holder what it measured; the facts now live on the bus: the
|
||||
// readings disk-load keeps (its state `readings`, with the item's seven-day peak), the last good night per item restic
|
||||
// keeps (its state `good`), and the manifests the indexer holds (for the bounds an item declares). Each kind, key and
|
||||
// word is the controller's (mesh-controller cmd/mesh-controller/data.go), so the condition the keeper raises from it is
|
||||
// the one the controller raised.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The states D13 reads (declared under `reads`).
|
||||
const (
|
||||
ReadingsRead = health.ReadingsRead
|
||||
GoodRead = health.GoodRead
|
||||
ModulesRead = delivery.IndexerModule + ".modules"
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
)
|
||||
|
||||
// The kinds D13 raises.
|
||||
const (
|
||||
kindDataShrank = "data-shrank"
|
||||
kindEmptyReplacement = "empty-replacement"
|
||||
kindDataHeldTwice = "data-held-twice"
|
||||
kindDataQuiet = "data-quiet"
|
||||
kindBackupStale = "backup-stale"
|
||||
kindDataUnmeasured = "data-unmeasured"
|
||||
kindDataMissing = "data-missing"
|
||||
kindArrayDegraded = "array-degraded"
|
||||
kindProtectionMissing = "protection-missing"
|
||||
kindCleanupWaiting = "cleanup-waiting"
|
||||
shrinkWindowDays = 7
|
||||
cleanupAfter = 30 * 24 * time.Hour
|
||||
shrinkFloor int64 = 16 << 20
|
||||
)
|
||||
|
||||
// dataFacts is everything D13 judges: the readings, the last good nights, each module's manifest, and the machines whose
|
||||
// backup holder was heard from.
|
||||
type dataFacts struct {
|
||||
readings []health.Reading
|
||||
good map[string]time.Time // by health.GoodKey
|
||||
manifests map[string]catalogue.Manifest
|
||||
// unmeasured are the machines that hold node-backup, are heard from, and have no reading of anything.
|
||||
unmeasured []string
|
||||
now time.Time
|
||||
}
|
||||
|
||||
// dataFacts reads them from the bus.
|
||||
func (c *Checker) dataFacts() (dataFacts, error) {
|
||||
f := dataFacts{good: map[string]time.Time{}, manifests: map[string]catalogue.Manifest{}, now: c.Now()}
|
||||
keys, err := c.Store.Keys(ReadingsRead)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("disk-load's readings cannot be read: %w", err)
|
||||
}
|
||||
byNode := map[string]int{}
|
||||
for _, k := range keys {
|
||||
raw, ok, err := c.Store.Get(ReadingsRead, k)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the reading %s cannot be read: %w", k, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
r, err := health.DecodeReading(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
f.readings = append(f.readings, r)
|
||||
byNode[r.Node]++
|
||||
}
|
||||
gkeys, err := c.Store.Keys(GoodRead)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("restic's last good nights cannot be read: %w", err)
|
||||
}
|
||||
for _, k := range gkeys {
|
||||
raw, ok, err := c.Store.Get(GoodRead, k)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the last good night %s cannot be read: %w", k, err)
|
||||
}
|
||||
if g, derr := health.DecodeGoodRow(raw); ok && derr == nil && !g.LastGood.IsZero() {
|
||||
f.good[health.GoodKey(g.Node, g.Module, g.Item)] = g.LastGood
|
||||
}
|
||||
}
|
||||
mods, err := c.Store.Keys(ModulesRead)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
for _, name := range mods {
|
||||
raw, ok, err := c.Store.Get(ModulesRead, name)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
// Only the manifest's digest of the indexer's module, and the manifest's text: what the bounds are read from.
|
||||
var m struct {
|
||||
Manifest string `json:"manifest"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
mraw, ok, err := c.Store.Get(ManifestsRead, m.Manifest)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
var doc struct {
|
||||
JSON string `json:"manifest"`
|
||||
}
|
||||
if json.Unmarshal(mraw, &doc) != nil {
|
||||
continue
|
||||
}
|
||||
var man catalogue.Manifest
|
||||
if json.Unmarshal([]byte(doc.JSON), &man) == nil {
|
||||
f.manifests[name] = man
|
||||
}
|
||||
}
|
||||
heard, err := c.heard()
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
holders, err := c.holdersOf("node-backup")
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, n := range holders {
|
||||
if heard[n] && byNode[n] == 0 {
|
||||
f.unmeasured = append(f.unmeasured, n)
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// probeData is D13.
|
||||
func probeData(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
f, err := c.dataFacts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dataFindings(f), nil
|
||||
}
|
||||
|
||||
func severityOf(class string) health.Severity {
|
||||
if class == catalogue.ClassIrreplaceable {
|
||||
return Urgent
|
||||
}
|
||||
return Warning
|
||||
}
|
||||
|
||||
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
|
||||
|
||||
func comparable(precision string) bool {
|
||||
return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ")
|
||||
}
|
||||
|
||||
func datasetOf(precision string) string {
|
||||
rest, ok := strings.CutPrefix(precision, "dataset ")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
name, _, _ := strings.Cut(rest, ":")
|
||||
return name
|
||||
}
|
||||
|
||||
func orUnknownPath(p string) string {
|
||||
if p == "" {
|
||||
return "a path its backup holder never named"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func sizeWords(size *int64) string {
|
||||
if size == nil || *size < 0 {
|
||||
return "size unknown"
|
||||
}
|
||||
b := float64(*size)
|
||||
for _, unit := range []string{"B", "KB", "MB", "GB"} {
|
||||
if b < 1024 || unit == "GB" {
|
||||
if unit == "B" {
|
||||
return fmt.Sprintf("%d B", *size)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", b, unit)
|
||||
}
|
||||
b /= 1024
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// firstSeen is when an item was first measured on its node (the reading keeps it); for a reading written before it did,
|
||||
// the oldest day its seven-day peak is made of, else its measurement: a lower bound on how long it has been declared.
|
||||
func firstSeen(r health.Reading, now time.Time) time.Time {
|
||||
if !r.FirstSeen.IsZero() {
|
||||
return r.FirstSeen
|
||||
}
|
||||
if len(r.Days) > 0 {
|
||||
if t, err := time.Parse("2006-01-02", r.Days[0].Day); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
if r.MeasuredAt != nil {
|
||||
return *r.MeasuredAt
|
||||
}
|
||||
return now
|
||||
}
|
||||
|
||||
func obs(kind string, id, machine string, sev health.Severity, summary, said string) Observation {
|
||||
return Observation{Kind: kind, Token: kind, Severity: sev, Summary: summary, Said: said,
|
||||
Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}}
|
||||
}
|
||||
|
||||
// dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was.
|
||||
func dataFindings(f dataFacts) []Observation {
|
||||
var out []Observation
|
||||
arrays := map[string][]health.Reading{}
|
||||
type shrunk struct {
|
||||
machine, dataset, class string
|
||||
size, peak int64
|
||||
items []string
|
||||
}
|
||||
shrunkDatasets := map[string]shrunk{}
|
||||
byItem := map[string][]health.Reading{}
|
||||
retiredCopies := map[string][]health.Reading{}
|
||||
for _, m := range f.unmeasured {
|
||||
o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+
|
||||
"declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data")
|
||||
o.Confirm = true
|
||||
out = append(out, o)
|
||||
}
|
||||
rs := append([]health.Reading(nil), f.readings...)
|
||||
sort.Slice(rs, func(i, j int) bool {
|
||||
return keyOfItem(rs[i].Node, rs[i].Module, rs[i].ID) < keyOfItem(rs[j].Node, rs[j].Module, rs[j].ID)
|
||||
})
|
||||
for _, r := range rs {
|
||||
class := r.Class
|
||||
item, declared := f.manifests[r.Module].DataItem(r.ID)
|
||||
id := keyOfItem(r.Node, r.Module, r.ID)
|
||||
if !r.RetiredAt.IsZero() {
|
||||
retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r)
|
||||
if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) {
|
||||
out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning,
|
||||
fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
|
||||
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
|
||||
r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24),
|
||||
r.RetiredWhy, r.Node, r.Module, r.ID, r.Module), "kept at "+orUnknownPath(r.Path)))
|
||||
out[len(out)-1].Token = "cleanup"
|
||||
}
|
||||
continue
|
||||
}
|
||||
byItem[r.Module+"/"+r.ID] = append(byItem[r.Module+"/"+r.ID], r)
|
||||
if !catalogue.Watched(class) {
|
||||
continue
|
||||
}
|
||||
severity := severityOf(class)
|
||||
if r.Redundancy != nil {
|
||||
where := r.Node + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
|
||||
arrays[where] = append(arrays[where], r)
|
||||
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.Error == "" {
|
||||
out = append(out, obs(kindProtectionMissing, id, r.Node, severity,
|
||||
fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
|
||||
"and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
|
||||
r.ID, r.Module, r.Node, class),
|
||||
orUnknownPath(r.Path)+" is on no redundant storage the backup holder can read"))
|
||||
}
|
||||
peak := r.Peak
|
||||
shrank := peak != nil && r.SizeBytes != nil && comparable(r.Precision) && *r.SizeBytes*2 < *peak && *peak-*r.SizeBytes >= shrinkFloor
|
||||
switch {
|
||||
case r.Error != "" && strings.Contains(r.Error, "does not exist"):
|
||||
out = append(out, obs(kindDataMissing, id, r.Node, severity,
|
||||
fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.ID, r.Module, r.Node, class),
|
||||
orUnknownPath(r.Path)+" does not exist: "+firstLine(r.Error)))
|
||||
case shrank && datasetOf(r.Precision) != "":
|
||||
// Several items on one dataset share its size: one condition for the dataset, as loud as the most
|
||||
// precious item on it.
|
||||
k := r.Node + "/" + datasetOf(r.Precision)
|
||||
ds := shrunkDatasets[k]
|
||||
ds.machine, ds.dataset, ds.size, ds.peak = r.Node, datasetOf(r.Precision), *r.SizeBytes, *peak
|
||||
ds.class = catalogue.StricterClass(ds.class, class)
|
||||
ds.items = append(ds.items, r.Module+"/"+r.ID)
|
||||
shrunkDatasets[k] = ds
|
||||
case shrank:
|
||||
out = append(out, obs(kindDataShrank, id, r.Node, severity,
|
||||
fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
|
||||
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
|
||||
"good copy beside it", r.ID, r.Module, r.Node, class, sizeWords(r.SizeBytes), sizeWords(peak), shrinkWindowDays), ""))
|
||||
}
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && f.now.Sub(*r.LastWrite) > within {
|
||||
out = append(out, obs(kindDataQuiet, id, r.Node, severity,
|
||||
fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
|
||||
"whatever writes it has stopped", r.ID, r.Module, r.Node, r.LastWrite.UTC().Format(time.RFC3339), within), ""))
|
||||
}
|
||||
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
|
||||
// plan, said only where the machine was measured — where a holder is there to take it.
|
||||
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
|
||||
within := item.BackupWithin()
|
||||
last, has := f.good[health.GoodKey(r.Node, r.Module, r.ID)]
|
||||
switch {
|
||||
case !has && f.now.Sub(firstSeen(r, f.now)) > within:
|
||||
out = append(out, obs(kindBackupStale, id, r.Node, severity,
|
||||
fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
|
||||
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
|
||||
r.ID, r.Module, r.Node, class, f.now.Sub(firstSeen(r, f.now)).Round(time.Hour)), ""))
|
||||
case has && f.now.Sub(last) > within:
|
||||
out = append(out, obs(kindBackupStale, id, r.Node, severity,
|
||||
fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
|
||||
"of %s", r.ID, r.Module, r.Node, class, last.UTC().Format(time.RFC3339), within), ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(shrunkDatasets))
|
||||
for k := range shrunkDatasets {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
ds := shrunkDatasets[k]
|
||||
out = append(out, obs(kindDataShrank, ds.machine+".dataset."+strings.ReplaceAll(ds.dataset, "/", "-"), ds.machine,
|
||||
severityOf(ds.class), fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than "+
|
||||
"half of what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak),
|
||||
shrinkWindowDays), "the dataset "+ds.dataset))
|
||||
}
|
||||
// The redundant storage watched data is on: one condition per array, as loud as the most precious item on it —
|
||||
// the array, not each item, is what degrades.
|
||||
wheres := make([]string, 0, len(arrays))
|
||||
for w := range arrays {
|
||||
wheres = append(wheres, w)
|
||||
}
|
||||
sort.Strings(wheres)
|
||||
for _, where := range wheres {
|
||||
items := arrays[where]
|
||||
red := items[0].Redundancy
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
continue
|
||||
}
|
||||
class, machine := catalogue.ClassValuable, items[0].Node
|
||||
var names []string
|
||||
for _, r := range items {
|
||||
class = catalogue.StricterClass(class, r.Class)
|
||||
names = append(names, r.Module+"/"+r.ID)
|
||||
}
|
||||
state := "could not be read"
|
||||
if red.Healthy != nil {
|
||||
state = "is NOT healthy"
|
||||
}
|
||||
out = append(out, obs(kindArrayDegraded, machine+".array."+strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
|
||||
machine, severityOf(class), fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine,
|
||||
strings.Join(names, ", "), state), fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))))
|
||||
}
|
||||
// The same item on several machines: a copy that is in use and far smaller than one kept retired elsewhere is an
|
||||
// empty replacement. Only against a retired copy — a module running on two machines on purpose keeps two different
|
||||
// sets of data.
|
||||
items := make([]string, 0, len(byItem))
|
||||
for k := range byItem {
|
||||
items = append(items, k)
|
||||
}
|
||||
sort.Strings(items)
|
||||
for _, key := range items {
|
||||
for _, a := range byItem[key] {
|
||||
if a.SizeBytes == nil || !catalogue.Watched(a.Class) || !comparable(a.Precision) {
|
||||
continue
|
||||
}
|
||||
for _, o := range retiredCopies[key] {
|
||||
if o.Node == a.Node || o.SizeBytes == nil || !comparable(o.Precision) || !replacedByLess(*a.SizeBytes, *o.SizeBytes) {
|
||||
continue
|
||||
}
|
||||
ob := obs(kindEmptyReplacement, keyOfItem(a.Node, a.Module, a.ID), a.Node, severityOf(a.Class),
|
||||
fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
|
||||
"an empty replacement of its data. Move the data, or assign it back where its data is",
|
||||
a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "")
|
||||
ob.Subject.Also = []string{o.Node}
|
||||
out = append(out, ob)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than half of it, and at
|
||||
// least shrinkFloor less.
|
||||
func replacedByLess(inUse, kept int64) bool { return inUse*2 < kept && kept-inUse >= shrinkFloor }
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
// D-fill (novox/hq ADR 0297 §6, issue 501): a module's bucket or log filling toward its cap is said before it is full.
|
||||
// Every module's bucket (`KV_<module>_<name>`) and log (`LOG_<module>_<name>`) is read from the bus module's
|
||||
// nats_streams, which answers each stream's size and its cap (max_bytes, mesh-catalog cab031bf). Said at three
|
||||
// quarters of its cap, cleared below seven tenths: between the two a finding already written is kept, so one at its
|
||||
// threshold is not raised and cleared on every run. The kind, key and words are the controller's
|
||||
// (mesh-controller bucket_log_fill.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// The fill thresholds, in percent of a bucket's or log's cap.
|
||||
const (
|
||||
fillRaiseAt = 75
|
||||
fillClearBelow = 70
|
||||
)
|
||||
|
||||
// fillRow is what nats_streams answers of a stream's size and cap.
|
||||
type fillRow struct {
|
||||
Name string `json:"name"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
MaxBytes uint64 `json:"max_bytes"`
|
||||
}
|
||||
|
||||
// bucketOrLog is a module's bucket or log by its stream: the module, `bucket` or `log`, and its local name. False for
|
||||
// a stream that is no module's bucket or log (the mesh's own streams).
|
||||
func bucketOrLog(stream string) (module, kind, name string, ok bool) {
|
||||
var rest string
|
||||
switch {
|
||||
case strings.HasPrefix(stream, "KV_"):
|
||||
kind, rest = "bucket", strings.TrimPrefix(stream, "KV_")
|
||||
case strings.HasPrefix(stream, buscompose.LogStreamPrefix):
|
||||
kind, rest = "log", strings.TrimPrefix(stream, buscompose.LogStreamPrefix)
|
||||
default:
|
||||
return "", "", "", false
|
||||
}
|
||||
module, name, ok = strings.Cut(rest, "_")
|
||||
if !ok || module == "" || name == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
return module, kind, name, true
|
||||
}
|
||||
|
||||
// mibWords is a size as a person reads it, in MiB with one decimal.
|
||||
func mibWords(b uint64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024)) }
|
||||
|
||||
// fillKey is where a bucket's or log's fill finding is kept.
|
||||
func fillKey(stream string) string {
|
||||
return Observation{Subject: Subject{Scope: ScopeBus, ID: stream}, Token: "filling"}.Key()
|
||||
}
|
||||
|
||||
// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and, while its finding
|
||||
// is written, at fillClearBelow or above too; nothing otherwise, which is what clears it.
|
||||
func fillObservation(stream string, r fillRow, open bool) (Observation, bool) {
|
||||
module, kind, name, ok := bucketOrLog(stream)
|
||||
if !ok || r.MaxBytes == 0 {
|
||||
return Observation{}, false // no module's, or with no cap: it cannot fill
|
||||
}
|
||||
// Compared in bytes, not rounded percent: 74.9% is not 75%.
|
||||
atRaise := r.Bytes*100 >= r.MaxBytes*fillRaiseAt
|
||||
aboveClear := r.Bytes*100 >= r.MaxBytes*fillClearBelow
|
||||
if !atRaise && !(open && aboveClear) {
|
||||
return Observation{}, false
|
||||
}
|
||||
pct := r.Bytes * 100 / r.MaxBytes
|
||||
return Observation{Kind: "bucket-or-log-filling", Token: "filling", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: stream},
|
||||
Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write",
|
||||
module, kind, name, mibWords(r.Bytes), mibWords(r.MaxBytes), pct),
|
||||
Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes,
|
||||
r.MaxBytes, pct),
|
||||
Headline: "A module's bucket or log on the bus is filling up",
|
||||
Needs: "decide whether to raise its cap or have the module keep less.",
|
||||
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it is " +
|
||||
"full, the bus refuses what the module writes there.",
|
||||
Resolved: "It has room again"}, true
|
||||
}
|
||||
|
||||
// probeFill is D-fill: every module's bucket and log on the bus judged against its cap. The bus module not answering
|
||||
// is the probe failing to run, which clears nothing; a finding already written counts as open.
|
||||
func probeFill(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
raw, err := c.Ask(StreamsTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err)
|
||||
}
|
||||
var a struct {
|
||||
Detail []fillRow `json:"detail"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err)
|
||||
}
|
||||
written, err := c.written()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its own findings cannot be read, so a fill between the thresholds cannot be judged: %w", err)
|
||||
}
|
||||
sort.Slice(a.Detail, func(i, j int) bool { return a.Detail[i].Name < a.Detail[j].Name })
|
||||
var out []Observation
|
||||
for _, r := range a.Detail {
|
||||
if o, said := fillObservation(r.Name, r, written.has(fillKey(r.Name))); said {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package main
|
||||
|
||||
// The probes of the mesh's machines (novox/hq ADR 0564 §6, issue 1952): each reads its facts where they live on the
|
||||
// bus — placement (the seat mesh-placement's state), who holds which seat (its seat-holders), whether a node is heard
|
||||
// from (the watchman's presence) — and asks each node's seat holder what only that node knows. Its assertion, kind,
|
||||
// key and words are the controller's (mesh-controller probes.go), so the condition the keeper raises from it is the one
|
||||
// the controller raised.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The states the machine probes read (declared under `reads`), and the verb D8 asks each node.
|
||||
const (
|
||||
PlacementRead = delivery.AssignerModule + "." + "placement"
|
||||
SeatHoldersRead = delivery.SeatHoldersRead
|
||||
PresenceRead = delivery.WatchmanModule + ".presence"
|
||||
BannedSeat = "node-intrusion-prevention"
|
||||
BannedVerb = "seat:" + BannedSeat + ".banned"
|
||||
ScopeMachine = health.ScopeMachine
|
||||
)
|
||||
|
||||
// placements is every node's placement, by node.
|
||||
func (c *Checker) placements() (map[string]delivery.Placement, error) {
|
||||
keys, err := c.Store.Keys(PlacementRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the placement cannot be read: %w", err)
|
||||
}
|
||||
out := map[string]delivery.Placement{}
|
||||
for _, k := range keys {
|
||||
raw, ok, err := c.Store.Get(PlacementRead, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the placement of %s cannot be read: %w", k, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var p delivery.Placement
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return nil, fmt.Errorf("the placement of %s is not readable: %w", k, err)
|
||||
}
|
||||
if p.Node == "" {
|
||||
p.Node = k
|
||||
}
|
||||
out[p.Node] = p
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heard is each node the watchman says is online.
|
||||
func (c *Checker) heard() (map[string]bool, error) {
|
||||
keys, err := c.Store.Keys(PresenceRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's presence cannot be read: %w", err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
node, _, _, err := delivery.ParsePresenceKey(k)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
raw, ok, err := c.Store.Get(PresenceRead, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the presence of %s cannot be read: %w", node, err)
|
||||
}
|
||||
var p delivery.Presence
|
||||
if ok && json.Unmarshal(raw, &p) == nil && p.Online {
|
||||
out[node] = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// holdersOf is each node on which a module holds the node seat seat, sorted.
|
||||
func (c *Checker) holdersOf(seat string) ([]string, error) {
|
||||
keys, err := c.Store.Keys(SeatHoldersRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("who holds which seat cannot be read: %w", err)
|
||||
}
|
||||
var out []string
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, seat+".") {
|
||||
continue
|
||||
}
|
||||
raw, ok, err := c.Store.Get(SeatHoldersRead, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the holders of %s cannot be read: %w", k, err)
|
||||
}
|
||||
var sh delivery.SeatHolders
|
||||
if !ok || json.Unmarshal(raw, &sh) != nil || sh.Seat != seat || sh.Node == "" || len(sh.Holders) == 0 {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(out, sh.Node) {
|
||||
out = append(out, sh.Node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// --- D8 ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||
|
||||
// lookupHost resolves an endpoint's name; replaced in tests.
|
||||
var lookupHost = func(ctx context.Context, host string) ([]string, error) {
|
||||
return net.DefaultResolver.LookupHost(ctx, host)
|
||||
}
|
||||
|
||||
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are every machine's
|
||||
// private address and the address its endpoint names, from placement; the ban lists are asked of each machine's holder
|
||||
// of node-intrusion-prevention that is heard from.
|
||||
func probeBans(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
places, err := c.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// address → whose, in machine names: what the summary says. The address, and the endpoint's name it was found
|
||||
// by, are the evidence (ADR 0234 §6).
|
||||
owned, via := map[string]string{}, map[string]string{}
|
||||
nodes := make([]string, 0, len(places))
|
||||
for n := range places {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
d := places[n].Decided
|
||||
if a, _, _ := strings.Cut(d.Address, "/"); a != "" {
|
||||
owned[a] = n + "'s private address"
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(d.Endpoint); err == nil && host != "" {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
owned[ip.String()] = n + "'s endpoint"
|
||||
} else if addrs, err := lookupHost(ctx, host); err == nil {
|
||||
for _, a := range addrs {
|
||||
owned[a], via[a] = n+"'s endpoint", host
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
heard, err := c.heard()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.holdersOf(BannedSeat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var holders []string
|
||||
for _, n := range all {
|
||||
if heard[n] {
|
||||
holders = append(holders, n)
|
||||
}
|
||||
}
|
||||
// Every machine asked at once: one after another, four holders that each wait their bound outlast the probe's
|
||||
// thirty seconds, and the probe says nothing about any of them.
|
||||
answers := make([]json.RawMessage, len(holders))
|
||||
errs := make([]error, len(holders))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range holders {
|
||||
wg.Add(1)
|
||||
go func(i int, node string) {
|
||||
defer wg.Done()
|
||||
answers[i], errs[i] = c.Ask(BannedVerb+"@"+node, map[string]any{})
|
||||
}(i, node)
|
||||
}
|
||||
wg.Wait()
|
||||
var out []Observation
|
||||
var unasked []string
|
||||
for i, node := range holders {
|
||||
if errs[i] != nil {
|
||||
unasked = append(unasked, errs[i].Error())
|
||||
continue
|
||||
}
|
||||
var banned, whose []string
|
||||
for _, ip := range ipLike.FindAllString(string(answers[i]), -1) {
|
||||
who, ours := owned[ip]
|
||||
said := ip + " (" + who + ")"
|
||||
if h := via[ip]; h != "" {
|
||||
said = ip + " (" + who + ", " + h + ")"
|
||||
}
|
||||
if !ours || slices.Contains(banned, said) {
|
||||
continue
|
||||
}
|
||||
banned = append(banned, said)
|
||||
if !slices.Contains(whose, who) {
|
||||
whose = append(whose, who)
|
||||
}
|
||||
}
|
||||
if len(banned) > 0 {
|
||||
out = append(out, Observation{Kind: "own-address-banned", Token: "bans-the-mesh", Severity: Urgent,
|
||||
Subject: Subject{Scope: ScopeMachine, ID: node, Machine: node},
|
||||
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es) — %s: the mesh is locked out "+
|
||||
"of itself there (ADR 0186)", node, len(banned), strings.Join(whose, ", ")),
|
||||
Said: strings.Join(banned, ", "),
|
||||
Headline: node + " has banned the mesh",
|
||||
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", node),
|
||||
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
||||
"mesh is locked out of it.", node),
|
||||
Resolved: node + " no longer bans the mesh"})
|
||||
}
|
||||
}
|
||||
if len(unasked) > 0 {
|
||||
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
// The account agents run as (novox/hq ADR 0266) and root where the trusted parties run (ADR 0259 §8), judged on the
|
||||
// machine's own word: each node-engine says, in its health report (mesh-host.health, key <node>), its verdict on every
|
||||
// account a module declares never to become root without a person. The judgement, its kind, key and words are the
|
||||
// controller's (mesh-controller agent_account.go). Whether the verdict is current is the watchman's presence: the
|
||||
// report is written on a change only, so a machine not heard from has no verdict on now.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The engine's words and bounds the judgement reads (mesh-host rootsearch).
|
||||
const (
|
||||
// HealthRead is the node-engines' health reports.
|
||||
HealthRead = delivery.NodeHealthRead
|
||||
// rootPending starts a verdict's reason while the engine's setuid search runs and no complete one judges.
|
||||
rootPending = "not judged yet (search running)"
|
||||
// rootQuiet is how long nothing is raised while an agent account waits for that search, counted from when this
|
||||
// self-check first saw it waiting: past it, waiting is itself the urgent condition. It never makes it confined.
|
||||
rootQuiet = 12 * time.Hour
|
||||
stateHealthy = "healthy"
|
||||
stateUnhealthy = "unhealthy"
|
||||
stateUnknown = "unknown"
|
||||
)
|
||||
|
||||
// report is a node's health report, and whether it has one.
|
||||
func (c *Checker) report(node string) (delivery.NodeHealth, bool, error) {
|
||||
raw, ok, err := c.Store.Get(HealthRead, node)
|
||||
if err != nil {
|
||||
return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if !ok {
|
||||
return delivery.NodeHealth{}, false, nil
|
||||
}
|
||||
h, err := delivery.DecodeNodeHealth(raw)
|
||||
if err != nil {
|
||||
return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s: %w", node, err)
|
||||
}
|
||||
return h, true, nil
|
||||
}
|
||||
|
||||
// pendingSince is, by node, when this self-check first saw its agent account waiting for the setuid search: kept while
|
||||
// it waits, forgotten once a verdict is complete. In this process alone: a restart of the node-engine does not restart
|
||||
// it, and a restart of the self-check gives a waiting account at most one more quiet.
|
||||
type pendingSince struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]time.Time
|
||||
}
|
||||
|
||||
func (p *pendingSince) since(node string, now time.Time) time.Time {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.seen == nil {
|
||||
p.seen = map[string]time.Time{}
|
||||
}
|
||||
if at, ok := p.seen[node]; ok {
|
||||
return at
|
||||
}
|
||||
p.seen[node] = now
|
||||
return now
|
||||
}
|
||||
|
||||
func (p *pendingSince) judged(node string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
delete(p.seen, node)
|
||||
}
|
||||
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
verdictOther = iota // anything else: not heard, an older engine, no verdict, another unknown
|
||||
verdictPending // waiting for the search, and otherwise healthy
|
||||
verdictComplete // judged: healthy, or a way to root found
|
||||
)
|
||||
|
||||
// judgedConfined is the controller's judgement over one machine's report: confined only when the machine is heard
|
||||
// from, its engine judges root, and every verdict on the agent account is healthy. why is said either way.
|
||||
func judgedConfined(agent string, h delivery.NodeHealth, had, heard bool) (bool, string) {
|
||||
switch {
|
||||
case !had:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+
|
||||
"nothing of what it runs", agent)
|
||||
case !heard:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine is not heard from, and a verdict "+
|
||||
"from before is not a verdict on now", agent)
|
||||
case h.Contract < delivery.RootContract:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+
|
||||
"the judging of an account's root (its statement's contract is %d, the judging is %d)",
|
||||
agent, h.Contract, delivery.RootContract)
|
||||
}
|
||||
verdicts := verdictsOn(agent, h)
|
||||
if len(verdicts) == 0 {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+
|
||||
"verdict on it — no module there declares it never to become root, or the declaration naming it "+
|
||||
"has not been applied", agent)
|
||||
}
|
||||
for _, v := range verdicts {
|
||||
switch v.State {
|
||||
case stateHealthy:
|
||||
case stateUnhealthy:
|
||||
return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent, orNoneKnown(v.Reason),
|
||||
v.Module, v.Resource)
|
||||
default:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent,
|
||||
orNoneKnown(v.Reason), v.Module, v.Resource, v.State)
|
||||
}
|
||||
}
|
||||
return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent,
|
||||
h.At.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
|
||||
// verdictsOn is every verdict on the account, sorted by module and resource.
|
||||
func verdictsOn(agent string, h delivery.NodeHealth) []delivery.RootVerdict {
|
||||
var out []delivery.RootVerdict
|
||||
for _, v := range h.Root {
|
||||
if v.Account == agent {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Resource < out[j].Module+out[j].Resource })
|
||||
return out
|
||||
}
|
||||
|
||||
// rootVerdictKind reads a report for the agent account: pending when every verdict on it is healthy or waits for the
|
||||
// setuid search, at least one of them that; complete when every verdict is healthy or one found a way to root.
|
||||
func rootVerdictKind(agent string, h delivery.NodeHealth, had, heard bool) int {
|
||||
if !had || !heard || h.Contract < delivery.RootContract {
|
||||
return verdictOther
|
||||
}
|
||||
pending, any := false, false
|
||||
for _, v := range verdictsOn(agent, h) {
|
||||
any = true
|
||||
switch {
|
||||
case v.State == stateHealthy:
|
||||
case v.State == stateUnhealthy:
|
||||
return verdictComplete
|
||||
case v.State == stateUnknown && strings.HasPrefix(v.Reason, rootPending):
|
||||
pending = true
|
||||
default:
|
||||
return verdictOther
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !any:
|
||||
return verdictOther
|
||||
case pending:
|
||||
return verdictPending
|
||||
}
|
||||
return verdictComplete
|
||||
}
|
||||
|
||||
// agentQuiet says the one thing keeping a machine's agent account unjudged is the engine's setuid search, first seen
|
||||
// waiting less than rootQuiet ago.
|
||||
func (c *Checker) agentQuiet(node, agent string, h delivery.NodeHealth, had, heard bool) bool {
|
||||
switch rootVerdictKind(agent, h, had, heard) {
|
||||
case verdictComplete:
|
||||
c.pending.judged(node)
|
||||
case verdictPending:
|
||||
now := c.clock()
|
||||
return now.Sub(c.pending.since(node, now)) <= rootQuiet
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// clock is the checker's clock, the wall clock where it has none.
|
||||
func (c *Checker) clock() time.Time {
|
||||
if c.Now != nil {
|
||||
return c.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// confinement is one machine's agent account judged: whether it names one, whether it is confined, why, and whether
|
||||
// it is unjudged only because its search runs within the quiet.
|
||||
type confinement struct {
|
||||
Named, Confined, Quiet bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// confined judges node's agent account from its placement, its report and its presence.
|
||||
func (c *Checker) confined(node string, place delivery.Placement, heard bool) (confinement, error) {
|
||||
agent := place.Decided.AgentAccount
|
||||
if agent == "" {
|
||||
return confinement{Why: fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)",
|
||||
node, orNoneKnown(place.Decided.Account))}, nil
|
||||
}
|
||||
h, had, err := c.report(node)
|
||||
if err != nil {
|
||||
return confinement{Named: true}, err
|
||||
}
|
||||
ok, why := judgedConfined(agent, h, had, heard)
|
||||
return confinement{Named: true, Confined: ok, Why: why, Quiet: !ok && c.agentQuiet(node, agent, h, had, heard)}, nil
|
||||
}
|
||||
|
||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest
|
||||
// statement, unable to become root without a person (ADR 0266).
|
||||
func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
places, err := c.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard, err := c.heard()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := make([]string, 0, len(places))
|
||||
for n := range places {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var out []Observation
|
||||
for _, n := range nodes {
|
||||
if places[n].Decided.AgentAccount == "" {
|
||||
continue
|
||||
}
|
||||
v, err := c.confined(n, places[n], heard[n])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Not judged yet only because the engine's search runs: not the urgent condition after every restart. Loud
|
||||
// again once the search fails, waits past the quiet, or the machine is not heard from.
|
||||
if v.Confined || v.Quiet {
|
||||
continue
|
||||
}
|
||||
out = append(out, Observation{Kind: "agent-can-become-root", Token: "agent-root", Severity: Urgent,
|
||||
Subject: Subject{Scope: ScopeMachine, ID: n, Machine: n},
|
||||
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+
|
||||
"answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why),
|
||||
Said: v.Why,
|
||||
Headline: "Sessions on " + n + " could become root",
|
||||
Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.",
|
||||
Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+
|
||||
"can take over the machine without you. The machine cannot show that this holds now, so an answer "+
|
||||
"from your phone authorises nothing there until it does.", n),
|
||||
Resolved: "Sessions on " + n + " cannot become root again"})
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// orNoneKnown is a value, or that none is known.
|
||||
func orNoneKnown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "none known"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed.
|
||||
type bus struct {
|
||||
drop map[string]bool // stream or stream.consumer missing
|
||||
change func(rows map[string]*streamRow)
|
||||
closed string
|
||||
signals error
|
||||
asksFail error
|
||||
// banned is each node's answer to node-intrusion-prevention.banned; a node absent does not answer.
|
||||
banned map[string]string
|
||||
// fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap.
|
||||
fills []fillRow
|
||||
}
|
||||
|
||||
func (b bus) rows() map[string]*streamRow {
|
||||
rows := map[string]*streamRow{}
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
if b.drop[s.Name] {
|
||||
continue
|
||||
}
|
||||
retention := "limits"
|
||||
per := int64(s.MaxMsgsPerSubject)
|
||||
switch s.Retention {
|
||||
case buscompose.RetentionWorkQueue:
|
||||
retention = "workqueue"
|
||||
case buscompose.RetentionLastPerSubject:
|
||||
per = 1
|
||||
}
|
||||
rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per}
|
||||
}
|
||||
for _, c := range buscompose.MeshConsumers() {
|
||||
r := rows[c.Stream]
|
||||
if r == nil || b.drop[c.Stream+"."+c.Name] {
|
||||
continue
|
||||
}
|
||||
kind := "pull"
|
||||
if c.Push || c.Queue != "" {
|
||||
kind = "push"
|
||||
}
|
||||
r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)})
|
||||
}
|
||||
if b.change != nil {
|
||||
b.change(rows)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func (b bus) ask(key string, body any) (json.RawMessage, error) {
|
||||
if b.asksFail != nil {
|
||||
return nil, b.asksFail
|
||||
}
|
||||
switch key {
|
||||
case StreamsTool:
|
||||
stream, _ := body.(map[string]any)["stream"].(string)
|
||||
var detail []streamRow
|
||||
for name, r := range b.rows() {
|
||||
if stream == "" {
|
||||
row := *r
|
||||
row.Consumers = nil
|
||||
detail = append(detail, row)
|
||||
} else if name == stream {
|
||||
detail = append(detail, *r)
|
||||
}
|
||||
}
|
||||
if stream == "" && len(b.fills) > 0 {
|
||||
var rows []any
|
||||
for _, r := range detail {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
for _, f := range b.fills {
|
||||
rows = append(rows, f)
|
||||
}
|
||||
return json.Marshal(map[string]any{"detail": rows})
|
||||
}
|
||||
if stream != "" && len(detail) == 0 {
|
||||
return nil, errors.New("no stream called " + stream)
|
||||
}
|
||||
return json.Marshal(map[string]any{"detail": detail})
|
||||
case ClosedTool:
|
||||
if b.closed == "" {
|
||||
return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil
|
||||
}
|
||||
return json.RawMessage(b.closed), nil
|
||||
case SignalsVerb:
|
||||
if b.signals != nil {
|
||||
return nil, b.signals
|
||||
}
|
||||
return json.RawMessage(`{"signals":[]}`), nil
|
||||
}
|
||||
if node, ok := strings.CutPrefix(key, BannedVerb+"@"); ok {
|
||||
if a, ok := b.banned[node]; ok {
|
||||
return json.RawMessage(a), nil
|
||||
}
|
||||
}
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
|
||||
func keysOf(found []Observation) []string {
|
||||
var out []string
|
||||
for _, f := range found {
|
||||
out = append(out, f.Key())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2).
|
||||
func TestTheBusProbes(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
probe func(context.Context, *Checker) ([]Observation, error)
|
||||
bus bus
|
||||
want []string
|
||||
err string
|
||||
mesh *memStore // the bus states the machine probes read; nil reads none
|
||||
}{
|
||||
{"D6 all as defined", probeConsumers, bus{}, nil, "", nil},
|
||||
{"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}},
|
||||
[]string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, "", nil},
|
||||
{"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}},
|
||||
[]string{"bus.CONTROL.controller.missing"}, "", nil},
|
||||
{"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) {
|
||||
for i, c := range rows["EVENTS"].Consumers {
|
||||
if c.Name == "controller" {
|
||||
rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000
|
||||
}
|
||||
}
|
||||
}}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, "", nil},
|
||||
{"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil},
|
||||
{"D7 all as defined", probeStreams, bus{}, nil, "", nil},
|
||||
{"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) {
|
||||
rows["EVENTS"].Retention = "workqueue"
|
||||
}}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, "", nil},
|
||||
{"D15 quiet", probeReconnects, bus{}, nil, "", nil},
|
||||
{"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[
|
||||
{"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]},
|
||||
{"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`},
|
||||
[]string{"bus.g14.node-tools.bus-reconnects"}, "", nil},
|
||||
{"DW answers", probeWatchdogs, bus{}, nil, "", nil},
|
||||
{"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, "", nil},
|
||||
{"H-bus healthy", probeBusHealth, bus{}, nil, "", nil},
|
||||
{"H-bus a stream and a consumer missing", probeBusHealth,
|
||||
bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, "", nil},
|
||||
{"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed", nil},
|
||||
// D8 (issue 238): placement's addresses and endpoints against each heard holder's ban list.
|
||||
{"D8 nothing of the mesh banned", probeBans, bus{banned: map[string]string{"ace": `{"banned":["198.51.100.7"]}`,
|
||||
"novox": `{"banned":[]}`}}, nil, "", machines()},
|
||||
{"D8 a private address and an endpoint banned", probeBans, bus{banned: map[string]string{
|
||||
"ace": `{"jails":{"sshd":["10.10.0.1","198.51.100.7"]}}`, "novox": `{"banned":["203.0.113.5"]}`}},
|
||||
[]string{"machine.ace.bans-the-mesh", "machine.novox.bans-the-mesh"}, "", machines()},
|
||||
{"D8 a machine not heard from is not asked", probeBans, bus{banned: map[string]string{"novox": `{"banned":[]}`,
|
||||
"ace": `{"banned":[]}`}}, nil, "", machines()},
|
||||
{"D8 a heard holder that does not answer is no pass", probeBans, bus{banned: map[string]string{}}, nil,
|
||||
"a ban list could not be read", machines()},
|
||||
// D13 (issue 273, ADR 0233): the readings disk-load keeps and the last good nights restic keeps, against the bounds
|
||||
// the manifests declare; the kinds, keys and words are the controller's.
|
||||
{"D13 all held", probeData, bus{}, nil, "", dataMesh(func(*dataRig) {})},
|
||||
{"D13 shrank, gone, quiet, unprotected and stale", probeData, bus{}, []string{
|
||||
"machine.ace.pg.data.data-quiet", "machine.ace.pg.data.backup-stale", "machine.ace.pg.dumps.data-missing",
|
||||
"machine.ace.plex.movies.protection-missing", "machine.ace.plex.thumbs.data-shrank",
|
||||
"machine.ace.array.md-md127.array-degraded"},
|
||||
"", dataMesh(func(r *dataRig) {
|
||||
r.size("ace", "pg", "data", 100<<20, 100<<20).write("ace", "pg", "data", -80*time.Hour)
|
||||
r.gone("ace", "pg", "dumps")
|
||||
r.size("ace", "plex", "thumbs", 10<<20, 600<<20)
|
||||
r.size("ace", "plex", "movies", 1<<30, 1<<30).noArray("ace", "plex", "movies")
|
||||
r.size("ace", "plex", "library", 1<<30, 1<<30).array("ace", "plex", "library", "md", "md127", false)
|
||||
r.lastGood("ace", "pg", "data", -100*time.Hour)
|
||||
})},
|
||||
{"D13 a dataset shrank once for its items", probeData, bus{}, []string{"machine.ace.dataset.storage-media.data-shrank"},
|
||||
"", dataMesh(func(r *dataRig) {
|
||||
r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media")
|
||||
r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media")
|
||||
})},
|
||||
{"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{
|
||||
"machine.ace.pg.dumps.cleanup"}, "", dataMesh(func(r *dataRig) {
|
||||
r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour)
|
||||
r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour)
|
||||
})},
|
||||
{"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{},
|
||||
[]string{"machine.ace.plex.thumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) {
|
||||
r.size("ace", "plex", "thumbs", 10<<20, 10<<20)
|
||||
r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour)
|
||||
})},
|
||||
{"D13 no good backup is counted from when the item was first measured", probeData, bus{},
|
||||
[]string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) {
|
||||
delete(r.good, health.ReadingKey("ace", "pg", "data"))
|
||||
delete(r.good, health.ReadingKey("ace", "plex", "meta"))
|
||||
r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour)
|
||||
})},
|
||||
{"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "",
|
||||
dataMesh(func(r *dataRig) { r.holder("g14") })},
|
||||
// D-fill (ADR 0297 §6): a module's bucket or log at three quarters of its cap, kept until under seven tenths.
|
||||
{"D-fill all with room", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 74, MaxBytes: 100},
|
||||
{Name: "LOG_mesh-delivery-builder_build-log", Bytes: 10, MaxBytes: 100}, {Name: "KV_nocap_x", Bytes: 999}}}, nil, "", nil},
|
||||
{"D-fill a bucket and a log filling", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 75, MaxBytes: 100},
|
||||
{Name: "LOG_mesh-delivery-builder_build-log", Bytes: 99, MaxBytes: 100}}},
|
||||
[]string{"bus.KV_mesh-issues_issues.filling", "bus.LOG_mesh-delivery-builder_build-log.filling"}, "", nil},
|
||||
{"D-fill between the thresholds raises nothing new", probeFill,
|
||||
bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, nil, "", nil},
|
||||
{"D-fill between the thresholds keeps what is said", probeFill,
|
||||
bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}},
|
||||
[]string{"bus.KV_mesh-issues_issues.filling"}, "", filling("KV_mesh-issues_issues")},
|
||||
{"D-fill the bus module silent", probeFill, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil},
|
||||
// DA (ADR 0266): the agent account each machine names, judged on its engine's report while it is heard from.
|
||||
{"DA confined", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": stateHealthy})},
|
||||
{"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "",
|
||||
agents(map[string]string{"novox": stateUnhealthy})},
|
||||
{"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "",
|
||||
agents(map[string]string{"novox": ""})},
|
||||
{"DA the search runs, within its quiet", probeAgentAccounts, bus{}, nil, "",
|
||||
agents(map[string]string{"novox": "pending"})},
|
||||
{"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-root"},
|
||||
"", agents(map[string]string{"novox": stateHealthy, "g14": stateHealthy})},
|
||||
{"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "",
|
||||
agents(map[string]string{"novox": "old"})},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c := &Checker{Ask: tc.bus.ask, Store: tc.mesh, Now: func() time.Time { return dataNow }}
|
||||
if tc.mesh == nil {
|
||||
c.Store = newStore()
|
||||
}
|
||||
found, err := tc.probe(context.Background(), c)
|
||||
if tc.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.err) {
|
||||
t.Errorf("%s: error %v, want %q", tc.name, err, tc.err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", tc.name, err)
|
||||
continue
|
||||
}
|
||||
if got := keysOf(found); !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%s: found %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// machines is a mesh of three: novox the hub with its endpoint, ace and g14 on the private network; ace and novox
|
||||
// hold node-intrusion-prevention and are heard from, g14 holds it and is not.
|
||||
func machines() *memStore {
|
||||
s := newStore()
|
||||
for _, p := range []struct {
|
||||
node, address, endpoint string
|
||||
online bool
|
||||
}{{"novox", "10.10.0.1/16", "203.0.113.5:51820", true}, {"ace", "10.10.0.2/16", "", true}, {"g14", "10.10.0.3/16", "", false}} {
|
||||
_ = s.Put(PlacementRead, p.node, delivery.Placement{V: 1, Node: p.node,
|
||||
Decided: delivery.NodeDecided{Address: p.address, Endpoint: p.endpoint, Hub: p.endpoint != ""}})
|
||||
_ = s.Put(PresenceRead, p.node, delivery.Presence{V: 1, Online: p.online})
|
||||
key, _ := delivery.SeatHoldersKey(BannedSeat, p.node)
|
||||
_ = s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: BannedSeat, Scope: "node", Node: p.node,
|
||||
Holders: []delivery.SeatHolder{{Module: "fail2ban", Node: p.node}}})
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// filling is a store holding the self-check's written fill finding of a stream: the condition open.
|
||||
func filling(stream string) *memStore {
|
||||
s := newStore()
|
||||
o := Observation{Kind: "bucket-or-log-filling", Token: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream},
|
||||
Summary: "filling", Source: "D-fill"}
|
||||
raw, _ := health.EncodeFinding(o.Finding("run", time.Now()))
|
||||
_ = s.Put(FindingsState, o.Key(), json.RawMessage(raw))
|
||||
return s
|
||||
}
|
||||
|
||||
// agents is machines() where each node named names the agent account "agent" and its report holds the verdict on it:
|
||||
// a state, "" for none, "pending" for one waiting for its search, "old" for an engine older than the judging.
|
||||
func agents(verdicts map[string]string) *memStore {
|
||||
s := machines()
|
||||
for node, v := range verdicts {
|
||||
raw, _, _ := s.Get(PlacementRead, node)
|
||||
var p delivery.Placement
|
||||
_ = json.Unmarshal(raw, &p)
|
||||
p.Decided.AgentAccount = "agent"
|
||||
_ = s.Put(PlacementRead, node, p)
|
||||
h := delivery.NodeHealth{V: 1, Node: node, Contract: 13}
|
||||
switch v {
|
||||
case "":
|
||||
case "old":
|
||||
h.Contract = 2
|
||||
h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: stateHealthy}}
|
||||
case "pending":
|
||||
h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent",
|
||||
State: stateUnknown, Reason: rootPending + ": 3 of 9 trees"}}
|
||||
default:
|
||||
h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: v,
|
||||
Reason: "can become root without a person: a sudo rule"}}
|
||||
}
|
||||
raw, _ = delivery.EncodeNodeHealth(h)
|
||||
_ = s.Put(HealthRead, node, json.RawMessage(raw))
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what
|
||||
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
|
||||
func TestTheRegistryIsADR0564sTable(t *testing.T) {
|
||||
moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard",
|
||||
"DW", "H-bus"}
|
||||
var ids []string
|
||||
for _, p := range Registry() {
|
||||
ids = append(ids, p.ID)
|
||||
if p.Kind == "" || p.Asserts == "" {
|
||||
t.Errorf("%s has no kind or assertion", p.ID)
|
||||
}
|
||||
if (p.Run == nil) == (p.Deferred == "") {
|
||||
t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID)
|
||||
}
|
||||
}
|
||||
slices.Sort(ids)
|
||||
slices.Sort(moved)
|
||||
if !slices.Equal(ids, moved) {
|
||||
t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- D13's rig ------------------------------------------------------------------------------------------------
|
||||
|
||||
var dataNow = time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// dataRig is the mesh D13 reads: a manifest per module with the items it declares, a reading per item, a last good night.
|
||||
type dataRig struct {
|
||||
s *memStore
|
||||
readings map[string]*health.Reading
|
||||
good map[string]time.Time
|
||||
}
|
||||
|
||||
func (r *dataRig) key(node, module, item string) string { return health.ReadingKey(node, module, item) }
|
||||
|
||||
func (r *dataRig) reading(node, module, item string) *health.Reading {
|
||||
k := r.key(node, module, item)
|
||||
if r.readings[k] == nil {
|
||||
at := dataNow
|
||||
r.readings[k] = &health.Reading{V: health.ReadingVersion, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module,
|
||||
ID: item, Class: catalogue.ClassValuable, Path: "/srv/" + module + "/" + item}, Measured: health.Measured{MeasuredAt: &at, Precision: "exact"}}
|
||||
}
|
||||
return r.readings[k]
|
||||
}
|
||||
|
||||
func (r *dataRig) size(node, module, item string, size, peak int64) *dataRig {
|
||||
x := r.reading(node, module, item)
|
||||
x.SizeBytes, x.Peak = &size, &peak
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) write(node, module, item string, ago time.Duration) *dataRig {
|
||||
t := dataNow.Add(ago)
|
||||
r.reading(node, module, item).LastWrite = &t
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) gone(node, module, item string) *dataRig {
|
||||
r.reading(node, module, item).Error = "/srv/" + module + "/" + item + " does not exist"
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) dataset(node, module, item, name string) *dataRig {
|
||||
r.reading(node, module, item).Precision = "dataset " + name + ": its whole size"
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) array(node, module, item, kind, where string, healthy bool) *dataRig {
|
||||
r.reading(node, module, item).Redundancy = &health.Redundancy{Kind: kind, Where: where, Healthy: &healthy, Said: "a member is missing"}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) noArray(node, module, item string) *dataRig {
|
||||
r.reading(node, module, item).Redundancy = nil
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) lastGood(node, module, item string, ago time.Duration) *dataRig {
|
||||
r.good[r.key(node, module, item)] = dataNow.Add(ago)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) retired(node, module, item string, ago time.Duration) *dataRig {
|
||||
x := r.reading(node, module, item)
|
||||
x.RetiredAt, x.RetiredWhy = dataNow.Add(ago), "no longer declared on "+node
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) firstSeen(node, module, item string, ago time.Duration) *dataRig {
|
||||
r.reading(node, module, item).FirstSeen = dataNow.Add(ago)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *dataRig) holder(node string) {
|
||||
key, _ := delivery.SeatHoldersKey("node-backup", node)
|
||||
_ = r.s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: "node-backup", Scope: "node", Node: node,
|
||||
Holders: []delivery.SeatHolder{{Module: "restic", Node: node}}})
|
||||
_ = r.s.Put(PresenceRead, node, delivery.Presence{V: 1, Online: true})
|
||||
}
|
||||
|
||||
// dataMesh builds the mesh: pg declares data (backed up within 48 h, written all the time within 24 h) and dumps; plex
|
||||
// declares movies (protected by redundancy), thumbs, meta and library (all valuable); every item is read as held, and what
|
||||
// is given changes it.
|
||||
func dataMesh(change func(*dataRig)) *memStore {
|
||||
r := &dataRig{s: newStore(), readings: map[string]*health.Reading{}, good: map[string]time.Time{}}
|
||||
r.holder("ace")
|
||||
declare := func(module string, items map[string]catalogue.DataItem) {
|
||||
var man catalogue.Manifest
|
||||
man.Module = module
|
||||
man.Data = &catalogue.Data{}
|
||||
for id, it := range items {
|
||||
it.ID = id
|
||||
man.Data.Own = append(man.Data.Own, it)
|
||||
}
|
||||
raw, _ := json.Marshal(man)
|
||||
digest := delivery.ManifestDigest(raw)
|
||||
_ = r.s.Put(ManifestsRead, digest, delivery.Manifest{V: 1, Module: module, JSON: string(raw)})
|
||||
_ = r.s.Put(ModulesRead, module, delivery.Module{V: 1, Manifest: digest})
|
||||
}
|
||||
declare("pg", map[string]catalogue.DataItem{
|
||||
"data": {Class: catalogue.ClassValuable, Within: "48h", Active: "24h"},
|
||||
"dumps": {Class: catalogue.ClassValuable}})
|
||||
declare("plex", map[string]catalogue.DataItem{
|
||||
"movies": {Class: catalogue.ClassValuable, Redundancy: "the array"},
|
||||
"thumbs": {Class: catalogue.ClassValuable},
|
||||
"meta": {Class: catalogue.ClassValuable},
|
||||
"library": {Class: catalogue.ClassValuable, Redundancy: "the array"}})
|
||||
for _, i := range [][2]string{{"pg", "data"}, {"pg", "dumps"}, {"plex", "movies"}, {"plex", "thumbs"}, {"plex", "meta"}, {"plex", "library"}} {
|
||||
r.size("ace", i[0], i[1], 1<<30, 1<<30).lastGood("ace", i[0], i[1], -time.Hour).write("ace", i[0], i[1], -time.Minute)
|
||||
if i[0] == "plex" && (i[1] == "movies" || i[1] == "library") {
|
||||
r.array("ace", i[0], i[1], "md", "md127", true)
|
||||
}
|
||||
}
|
||||
change(r)
|
||||
for k, x := range r.readings {
|
||||
_ = r.s.Put(ReadingsRead, k, x)
|
||||
}
|
||||
for k, t := range r.good {
|
||||
node, rest, _ := strings.Cut(k, ".")
|
||||
module, item, _ := strings.Cut(rest, ".")
|
||||
_ = r.s.Put(GoodRead, k, health.GoodRow{V: health.GoodVersion, Node: node, Module: module, Item: item, CoveredBy: "/x", LastGood: t})
|
||||
}
|
||||
return r.s
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-self-check/module.json
|
||||
@@ -0,0 +1,417 @@
|
||||
package main
|
||||
|
||||
// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five
|
||||
// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket
|
||||
// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the
|
||||
// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the
|
||||
// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is
|
||||
// said in the verdict at once and written as a finding of its own when the next run cannot run it either.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Module and Seat are this module and the seat it holds once for the mesh.
|
||||
const (
|
||||
Module = "mesh-self-check"
|
||||
Seat = "mesh-self-check"
|
||||
)
|
||||
|
||||
// The states it keeps, and the event that ends every run.
|
||||
const (
|
||||
FindingsState = health.FindingsState
|
||||
VerdictState = "verdict"
|
||||
VerdictsState = "verdicts"
|
||||
LastKey = "last"
|
||||
CheckedEvent = health.EventChecked
|
||||
// CheckedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
|
||||
// consumers hear (catalogue.SeatSays).
|
||||
CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent
|
||||
)
|
||||
|
||||
// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1).
|
||||
var (
|
||||
Every = 5 * time.Minute
|
||||
FirstAfter = time.Minute
|
||||
Within = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have, as mesh-sdk's health names them.
|
||||
const (
|
||||
VerdictPass = health.VerdictPassed
|
||||
VerdictFail = health.VerdictFailed
|
||||
VerdictFailedToRun = health.VerdictFailedToRun
|
||||
VerdictDeferred = health.VerdictDeferred
|
||||
)
|
||||
|
||||
// sourceSelf is what writes a probe's failure to run: the self-check itself.
|
||||
const sourceSelf = "doctor"
|
||||
|
||||
// Store is the bus's buckets as this module reaches them, through its runtime.
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, bool, error)
|
||||
Put(state, key string, value any) error
|
||||
Delete(state, key string) error
|
||||
Keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// On says whether the setting lists the probe.
|
||||
func (c *Checker) On(id string) bool {
|
||||
if c.Enabled == nil {
|
||||
return true
|
||||
}
|
||||
return slices.Contains(c.Enabled(), id)
|
||||
}
|
||||
|
||||
// DefaultProbes are the probes that run unless the setting says otherwise.
|
||||
var DefaultProbes = []string{"D13", "D-fill"}
|
||||
|
||||
// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits.
|
||||
type Probe struct {
|
||||
ID string
|
||||
Asserts string
|
||||
From string
|
||||
Kind string
|
||||
Raises []string
|
||||
// Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6):
|
||||
// answered deferred, never a pass.
|
||||
Deferred string
|
||||
Run func(ctx context.Context, c *Checker) ([]Observation, error)
|
||||
}
|
||||
|
||||
// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's
|
||||
// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it.
|
||||
type (
|
||||
ProbeVerdict = health.ProbeVerdict
|
||||
Counts = health.CheckedCounts
|
||||
Run = health.Checked
|
||||
)
|
||||
|
||||
// Checker is the self-check: the registry and what its probes reach.
|
||||
type Checker struct {
|
||||
Store Store
|
||||
// Enabled is the setting `probes`: the ids of the probes that run. Nil runs every probe in the registry; one not
|
||||
// listed runs nothing and raises nothing (the operator, 2026-10-30).
|
||||
Enabled func() []string
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
|
||||
Publish func(subject string, body any, id string) error
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
Host string
|
||||
Registry []Probe
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
n uint64
|
||||
last *Run
|
||||
// seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is
|
||||
// written only when two runs in a row see it, or while it is already written (novox/hq issue 277).
|
||||
seen map[string]bool
|
||||
// unsaid counts the runs whose `checked` could not be said since the start.
|
||||
unsaid int
|
||||
// pending is when each machine's agent account was first seen waiting for its setuid search (DA, D-root).
|
||||
pending pendingSince
|
||||
}
|
||||
|
||||
// LastRun is the last run, nil before the first.
|
||||
func (c *Checker) LastRun() *Run {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.last
|
||||
}
|
||||
|
||||
type result struct {
|
||||
found []Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`.
|
||||
// One run at a time: a person's run during a scheduled one waits for it.
|
||||
func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
c.running.Lock()
|
||||
defer c.running.Unlock()
|
||||
c.mu.Lock()
|
||||
c.n++
|
||||
n := c.n
|
||||
c.mu.Unlock()
|
||||
started := c.Now()
|
||||
run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||
IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why}
|
||||
|
||||
results := make([]result, len(c.Registry))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range c.Registry {
|
||||
if p.Run == nil || !c.On(p.ID) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, p Probe) {
|
||||
defer wg.Done()
|
||||
results[i] = c.probe(ctx, p)
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
written, readErr := c.written()
|
||||
seen := map[string]bool{}
|
||||
want := map[string]Observation{} // every finding this run writes, by key
|
||||
ran := map[string]bool{} // the sources whose findings this run judged whole
|
||||
var blind []Observation
|
||||
for i, p := range c.Registry {
|
||||
if !c.On(p.ID) {
|
||||
// Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no
|
||||
// verdict in the run.
|
||||
ran[p.ID] = true
|
||||
continue
|
||||
}
|
||||
v := ProbeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
switch {
|
||||
case p.Run == nil:
|
||||
v.Verdict = VerdictDeferred
|
||||
v.Error = "deferred: " + p.Deferred
|
||||
run.Counts.Deferred++
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = VerdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
ran[p.ID] = true
|
||||
for _, f := range r.found {
|
||||
if f.Kind == "" {
|
||||
f.Kind = p.Kind
|
||||
}
|
||||
f.Source = p.ID
|
||||
f = worded(f)
|
||||
key := f.Key()
|
||||
seen[key] = true
|
||||
if f.Confirm && !c.seenBefore(key) && !written.has(key) {
|
||||
v.Unconfirmed = append(v.Unconfirmed, f.Summary)
|
||||
continue
|
||||
}
|
||||
want[key] = f
|
||||
v.Found = append(v.Found, f.Summary)
|
||||
}
|
||||
if len(v.Found) == 0 {
|
||||
v.Verdict = VerdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = VerdictFail
|
||||
run.Counts.Failed++
|
||||
}
|
||||
}
|
||||
if p.Run != nil {
|
||||
v.Took = r.took.Round(time.Millisecond).String()
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
ran[sourceSelf] = true
|
||||
for _, f := range blind {
|
||||
f = worded(f)
|
||||
key := f.Key()
|
||||
seen[key] = true
|
||||
if c.seenBefore(key) || written.has(key) {
|
||||
want[key] = f
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
// Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so.
|
||||
c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr)
|
||||
} else if err := c.write(written, want, ran, run.Run); err != nil {
|
||||
c.Logf("[%s] what this run found is not all written: %v", Module, err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.seen = seen
|
||||
c.mu.Unlock()
|
||||
|
||||
run.At = c.Now().UTC()
|
||||
run.Took = run.At.Sub(started).Round(time.Millisecond).String()
|
||||
c.mu.Lock()
|
||||
run.Unsaid = c.unsaid
|
||||
c.mu.Unlock()
|
||||
c.keep(run)
|
||||
return run
|
||||
}
|
||||
|
||||
// probe runs one probe within its bound; a panic or an overrun is an error, never a pass.
|
||||
func (c *Checker) probe(ctx context.Context, p Probe) result {
|
||||
probing, cancel := context.WithTimeout(ctx, Within)
|
||||
defer cancel()
|
||||
began := c.Now()
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
found, err := p.Run(probing, c)
|
||||
done <- result{found: found, err: err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
r.took = c.Now().Sub(began)
|
||||
return r
|
||||
case <-probing.Done():
|
||||
return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) seenBefore(key string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.seen[key]
|
||||
}
|
||||
|
||||
// writtenFindings are the findings in the bucket, by key.
|
||||
type writtenFindings map[string]health.Finding
|
||||
|
||||
func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok }
|
||||
|
||||
// written reads every finding this module has written.
|
||||
func (c *Checker) written() (writtenFindings, error) {
|
||||
keys, err := c.Store.Keys(FindingsState)
|
||||
if err != nil {
|
||||
return writtenFindings{}, err
|
||||
}
|
||||
out := writtenFindings{}
|
||||
for _, k := range keys {
|
||||
raw, found, err := c.Store.Get(FindingsState, k)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the finding %s is not readable: %w", k, err)
|
||||
}
|
||||
out[k] = f
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding
|
||||
// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4).
|
||||
func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error {
|
||||
var errs []string
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
now := c.Now()
|
||||
for _, k := range keys {
|
||||
f := want[k].Finding(run, now)
|
||||
if was, ok := written[k]; ok && says(was) == says(f) {
|
||||
continue
|
||||
}
|
||||
raw, err := health.EncodeFinding(f)
|
||||
if err == nil {
|
||||
err = c.Store.Put(FindingsState, k, json.RawMessage(raw))
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, k+": "+err.Error())
|
||||
}
|
||||
}
|
||||
for k, f := range written {
|
||||
if _, still := want[k]; still || !ran[f.Source] {
|
||||
continue
|
||||
}
|
||||
if err := c.Store.Delete(FindingsState, k); err != nil {
|
||||
errs = append(errs, k+": "+err.Error())
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in
|
||||
// the log: the watchman's S10 says the self-check silent when `checked` stops.
|
||||
func (c *Checker) keep(run Run) {
|
||||
c.mu.Lock()
|
||||
c.last = &run
|
||||
c.mu.Unlock()
|
||||
if err := c.Store.Put(VerdictState, LastKey, run); err != nil {
|
||||
c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err)
|
||||
}
|
||||
if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil {
|
||||
c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err)
|
||||
}
|
||||
if c.Publish == nil {
|
||||
return
|
||||
}
|
||||
if err := c.Publish(CheckedSubject, run, run.Run); err != nil {
|
||||
c.mu.Lock()
|
||||
c.unsaid++
|
||||
c.mu.Unlock()
|
||||
c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v",
|
||||
Module, run.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314).
|
||||
const overviewFindings = 20
|
||||
|
||||
// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all.
|
||||
func Overview(run Run) Run {
|
||||
probes := make([]ProbeVerdict, len(run.Probes))
|
||||
for i, p := range run.Probes {
|
||||
total := len(p.Found) + len(p.Unconfirmed)
|
||||
if total > overviewFindings {
|
||||
p.FoundCount = len(p.Found)
|
||||
if len(p.Found) > overviewFindings {
|
||||
p.Found = p.Found[:overviewFindings:overviewFindings]
|
||||
}
|
||||
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
|
||||
p.Unconfirmed = p.Unconfirmed[:room:room]
|
||||
}
|
||||
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all",
|
||||
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID)
|
||||
}
|
||||
probes[i] = p
|
||||
}
|
||||
run.Probes = probes
|
||||
return run
|
||||
}
|
||||
|
||||
// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every.
|
||||
func (c *Checker) Keep(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(FirstAfter):
|
||||
}
|
||||
tick := time.NewTicker(Every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
c.RunOnce(ctx, "the schedule")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(s, "\n")
|
||||
return line
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// memStore is the bus's buckets in memory, counting the writes.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
kv map[string]map[string][]byte
|
||||
puts map[string]int
|
||||
keysErr error
|
||||
}
|
||||
|
||||
func newStore() *memStore {
|
||||
return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}}
|
||||
}
|
||||
|
||||
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.kv[state][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Put(state, key string, value any) error {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.kv[state] == nil {
|
||||
s.kv[state] = map[string][]byte{}
|
||||
}
|
||||
s.kv[state][key] = raw
|
||||
s.puts[state+"/"+key]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Delete(state, key string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.kv[state], key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Keys(state string) ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.keysErr != nil && state == FindingsState {
|
||||
return nil, s.keysErr
|
||||
}
|
||||
var out []string
|
||||
for k := range s.kv[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *memStore) findingKeys() []string {
|
||||
keys, _ := s.Keys(FindingsState)
|
||||
return keys
|
||||
}
|
||||
|
||||
// checker is a self-check over memory, with a registry the test drives.
|
||||
func checker(store *memStore, registry ...Probe) (*Checker, *[]string) {
|
||||
var said []string
|
||||
now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { now = now.Add(time.Second); return now },
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
said = append(said, subject+" "+id+" "+string(raw))
|
||||
return nil
|
||||
}}
|
||||
return c, &said
|
||||
}
|
||||
|
||||
// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that
|
||||
// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can
|
||||
// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1).
|
||||
func TestARunWritesFindingsAsTheyChange(t *testing.T) {
|
||||
var found []Observation
|
||||
var fail error
|
||||
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }}
|
||||
deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"}
|
||||
missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"},
|
||||
Summary: "the stream EVENTS is not on the bus"}
|
||||
behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"}
|
||||
store := newStore()
|
||||
c, said := checker(store, steady, deferred)
|
||||
|
||||
steps := []struct {
|
||||
name string
|
||||
found []Observation
|
||||
fail error
|
||||
want []string // finding keys written after the run
|
||||
verdict string
|
||||
missingPs int // puts of the missing finding so far
|
||||
}{
|
||||
{"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"seen twice: written", []Observation{missing, behind}, nil,
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"blind once: clears nothing, held", nil, errors.New("timed out"),
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1},
|
||||
{"blind twice: its own finding", nil, errors.New("timed out"),
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1},
|
||||
{"no longer seen: cleared", nil, nil, nil, VerdictPass, 1},
|
||||
}
|
||||
for _, s := range steps {
|
||||
found, fail = s.found, s.fail
|
||||
run := c.RunOnce(context.Background(), "the test")
|
||||
if got := store.findingKeys(); !slices.Equal(got, s.want) {
|
||||
t.Errorf("%s: findings %v, want %v", s.name, got, s.want)
|
||||
}
|
||||
if run.Probes[0].Verdict != s.verdict {
|
||||
t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict)
|
||||
}
|
||||
if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") {
|
||||
t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1])
|
||||
}
|
||||
if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs {
|
||||
t.Errorf("%s: the unchanged finding was written %d times", s.name, n)
|
||||
}
|
||||
}
|
||||
raw, _, _ := store.Get(FindingsState, "probe.P1.failed")
|
||||
if raw != nil {
|
||||
t.Errorf("the blind finding outlived the probe running again: %s", raw)
|
||||
}
|
||||
// Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body.
|
||||
if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) {
|
||||
t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps))
|
||||
}
|
||||
// Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id.
|
||||
if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") {
|
||||
t.Fatalf("said %v", *said)
|
||||
}
|
||||
_, rest, _ := strings.Cut((*said)[0], " ")
|
||||
_, rest, _ = strings.Cut(rest, " ")
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal([]byte(rest), &body)
|
||||
for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} {
|
||||
if _, ok := body[field]; !ok {
|
||||
t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body)
|
||||
}
|
||||
}
|
||||
last, _, _ := store.Get(VerdictState, LastKey)
|
||||
var r Run
|
||||
_ = json.Unmarshal(last, &r)
|
||||
if r.Counts.Passed != 1 || r.Counts.Deferred != 1 {
|
||||
t.Errorf("the last verdict counts %+v", r.Counts)
|
||||
}
|
||||
// A finding carries the operator's words of its kind, the probe that found it and who observed it.
|
||||
found, fail = []Observation{missing}, nil
|
||||
c.RunOnce(context.Background(), "the test")
|
||||
raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing")
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" ||
|
||||
f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" {
|
||||
t.Errorf("the written finding is %+v", f)
|
||||
}
|
||||
// Blind to its own bucket, a run writes and clears nothing.
|
||||
store.keysErr = errors.New("the bus is away")
|
||||
found = nil
|
||||
c.RunOnce(context.Background(), "the test")
|
||||
store.keysErr = nil
|
||||
if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) {
|
||||
t.Errorf("a run blind to its findings changed them: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A probe that panics or overruns its bound could not run: never a pass.
|
||||
func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
|
||||
within := Within
|
||||
Within = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Within = within })
|
||||
for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){
|
||||
"panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") },
|
||||
"overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil },
|
||||
} {
|
||||
c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run})
|
||||
c.Now = time.Now
|
||||
r := c.RunOnce(context.Background(), "the test")
|
||||
if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 {
|
||||
t.Errorf("%s: %+v", name, r.Probes[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer
|
||||
// every state the module keeps (ADR 0564 §12).
|
||||
func TestTheVerbsAndReads(t *testing.T) {
|
||||
store := newStore()
|
||||
many := make([]Observation, 25)
|
||||
for i := range many {
|
||||
many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"}
|
||||
}
|
||||
c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong",
|
||||
Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }})
|
||||
if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") {
|
||||
t.Fatalf("before a run: %v", err)
|
||||
}
|
||||
if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over, _ := c.Doctor(false, "", "")
|
||||
if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" {
|
||||
t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More)
|
||||
}
|
||||
whole, _ := c.Doctor(false, "d7", "")
|
||||
if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 {
|
||||
t.Errorf("one probe whole carries %d findings", len(v.Found))
|
||||
}
|
||||
if _, err := c.Doctor(false, "D99", ""); err == nil {
|
||||
t.Error("an unknown probe answered")
|
||||
}
|
||||
// A fresh checker over the same bucket answers the last verdict from it.
|
||||
fresh, _ := checker(store)
|
||||
if _, err := fresh.Doctor(false, "", ""); err != nil {
|
||||
t.Errorf("the last verdict is not read from its bucket: %v", err)
|
||||
}
|
||||
fs, err := c.Findings("")
|
||||
if err != nil || fs.(map[string]any)["count"] != 25 {
|
||||
t.Errorf("findings answered %v, %v", fs, err)
|
||||
}
|
||||
if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") {
|
||||
t.Errorf("one finding answered %v, %v", one, err)
|
||||
}
|
||||
vs, err := c.Verdicts("", 0)
|
||||
if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 {
|
||||
t.Errorf("verdicts answered %v, %v", vs, err)
|
||||
}
|
||||
if _, err := c.Verdicts("doctor-1-1", 0); err == nil {
|
||||
t.Error("an unknown run answered")
|
||||
}
|
||||
if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" {
|
||||
t.Errorf("probes answered %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the probes the setting `probes` lists run: one not listed runs nothing, has no verdict, clears what it raised before,
|
||||
// and is reported by the verb as off by setting; the default list is D13 and D-fill (the operator, 2026-10-30).
|
||||
func TestOnlyTheProbesTheSettingListsRun(t *testing.T) {
|
||||
ran := map[string]int{}
|
||||
finds := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "gone"}
|
||||
probe := func(id string) Probe {
|
||||
return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) {
|
||||
ran[id]++
|
||||
return []Observation{finds}, nil
|
||||
}}
|
||||
}
|
||||
store := newStore()
|
||||
c, _ := checker(store, probe("D7"), probe("D13"))
|
||||
c.Enabled = func() []string { return []string{"D7", "D13"} }
|
||||
run := c.RunOnce(context.Background(), "test")
|
||||
if ran["D7"] != 1 || ran["D13"] != 1 || len(run.Probes) != 2 || len(store.findingKeys()) != 1 {
|
||||
t.Fatalf("both listed: ran %v, %d verdicts, findings %v", ran, len(run.Probes), store.findingKeys())
|
||||
}
|
||||
c.Enabled = func() []string { return []string{"D13"} }
|
||||
run = c.RunOnce(context.Background(), "test")
|
||||
if ran["D7"] != 1 || ran["D13"] != 2 || len(run.Probes) != 1 || run.Probes[0].ID != "D13" {
|
||||
t.Fatalf("D7 unlisted: ran %v, verdicts %+v", ran, run.Probes)
|
||||
}
|
||||
// What the unlisted probe raised stays only as long as another listed probe sees the same key (D13 raises the same
|
||||
// key here); an unlisted probe alone clears its findings.
|
||||
c.Enabled = func() []string { return nil }
|
||||
c.RunOnce(context.Background(), "test")
|
||||
if len(store.findingKeys()) != 0 {
|
||||
t.Fatalf("with nothing listed, findings are %v", store.findingKeys())
|
||||
}
|
||||
rows, _ := c.Probes()["probes"].([]map[string]any)
|
||||
if len(rows) != 2 || rows[0]["off"] != "off by setting" || rows[1]["off"] != "off by setting" {
|
||||
t.Fatalf("the verb says %v", rows)
|
||||
}
|
||||
c.Enabled = nil
|
||||
if !c.On("anything") || !slices.Equal(DefaultProbes, []string{"D13", "D-fill"}) {
|
||||
t.Fatal("no setting runs every probe, and the default list is D13 and D-fill")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package main
|
||||
|
||||
// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once,
|
||||
// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the
|
||||
// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Verbs are the seat's verbs, as the manifest claims them.
|
||||
var Verbs = []string{"doctor", "probes"}
|
||||
|
||||
// Tool names: the seat's verbs, and the module's own reads.
|
||||
const (
|
||||
DoctorTool = Seat + ".doctor"
|
||||
ProbesTool = Seat + ".probes"
|
||||
FindingsTool = "self_check_findings"
|
||||
VerdictsTool = "self_check_verdicts"
|
||||
)
|
||||
|
||||
func obj(props map[string]any) map[string]any {
|
||||
return map[string]any{"type": "object", "properties": props}
|
||||
}
|
||||
|
||||
// Tools are what the runtime serves.
|
||||
func Tools(c *Checker) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: DoctorTool,
|
||||
Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " +
|
||||
"failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " +
|
||||
"every probe now and answers that run; probe=<id> gives one probe's verdict whole, every finding.",
|
||||
Input: obj(map[string]any{
|
||||
"run": map[string]any{"type": "boolean", "description": "run every probe now"},
|
||||
"probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a))
|
||||
}},
|
||||
{Name: ProbesTool,
|
||||
Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " +
|
||||
"it raises, and for one deferred the facts it lacks on the bus.",
|
||||
Run: func(map[string]any) (any, error) { return c.Probes(), nil }},
|
||||
{Name: FindingsTool,
|
||||
Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " +
|
||||
"finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.",
|
||||
Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}),
|
||||
Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }},
|
||||
{Name: VerdictsTool,
|
||||
Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " +
|
||||
"with its counts; or one run whole by its id. The last run is the doctor verb's.",
|
||||
Input: obj(map[string]any{
|
||||
"run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"},
|
||||
"limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}),
|
||||
Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }},
|
||||
}
|
||||
}
|
||||
|
||||
// Doctor is the verb: the last verdict, a run now, or one probe of either.
|
||||
func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) {
|
||||
var r Run
|
||||
switch {
|
||||
case run:
|
||||
r = c.RunOnce(context.Background(), why)
|
||||
case c.LastRun() != nil:
|
||||
r = *c.LastRun()
|
||||
default:
|
||||
raw, found, err := c.Store.Get(VerdictState, LastKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the last verdict cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+
|
||||
"every %s — run=true runs it now", FirstAfter, Every)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("the last verdict is not readable: %w", err)
|
||||
}
|
||||
}
|
||||
if probe != "" {
|
||||
one, ok := oneProbe(r, probe)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool)
|
||||
}
|
||||
r = one
|
||||
} else {
|
||||
r = Overview(r)
|
||||
}
|
||||
return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " +
|
||||
"longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil
|
||||
}
|
||||
|
||||
func oneProbe(r Run, id string) (Run, bool) {
|
||||
for _, p := range r.Probes {
|
||||
if strings.EqualFold(p.ID, id) {
|
||||
r.Probes = []ProbeVerdict{p}
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return Run{}, false
|
||||
}
|
||||
|
||||
// Probes is the registry as the verb answers it.
|
||||
func (c *Checker) Probes() map[string]any {
|
||||
var out []map[string]any
|
||||
for _, p := range c.Registry {
|
||||
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind}
|
||||
if len(p.Raises) > 0 {
|
||||
row["raises"] = p.Raises
|
||||
}
|
||||
if !c.On(p.ID) {
|
||||
row["off"] = "off by setting"
|
||||
} else if p.Run == nil {
|
||||
row["deferred"] = p.Deferred
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()}
|
||||
}
|
||||
|
||||
// Findings are the findings written now, or one whole.
|
||||
func (c *Checker) Findings(key string) (any, error) {
|
||||
if key != "" {
|
||||
raw, found, err := c.Store.Get(FindingsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no finding is written under %s now", key)
|
||||
}
|
||||
return json.RawMessage(raw), nil
|
||||
}
|
||||
written, err := c.written()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the findings cannot be read: %w", err)
|
||||
}
|
||||
rows := []map[string]any{}
|
||||
for k, f := range written {
|
||||
rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source,
|
||||
"summary": f.Summary})
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) })
|
||||
return map[string]any{"findings": rows, "count": len(rows)}, nil
|
||||
}
|
||||
|
||||
// Verdicts are the week's runs newest first, or one whole.
|
||||
func (c *Checker) Verdicts(run string, limit int) (any, error) {
|
||||
if run != "" {
|
||||
raw, found, err := c.Store.Get(VerdictsState, run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run)
|
||||
}
|
||||
return json.RawMessage(raw), nil
|
||||
}
|
||||
keys, err := c.Store.Keys(VerdictsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) })
|
||||
total := len(keys)
|
||||
if limit > 0 && len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
}
|
||||
rows := []map[string]any{}
|
||||
for _, k := range keys {
|
||||
raw, found, err := c.Store.Get(VerdictsState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
var r Run
|
||||
if json.Unmarshal(raw, &r) == nil {
|
||||
rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts})
|
||||
}
|
||||
}
|
||||
return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil
|
||||
}
|
||||
|
||||
// runOrder orders run ids `doctor-<unix>-<n>` by time, then number.
|
||||
func runOrder(id string) string {
|
||||
parts := strings.Split(id, "-")
|
||||
if len(parts) != 3 {
|
||||
return id
|
||||
}
|
||||
return fmt.Sprintf("%020s-%020s", parts[1], parts[2])
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, k string) string {
|
||||
s, _ := a[k].(string)
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// boolArg reads a flag as a boolean or as the seat's "true".
|
||||
func boolArg(a map[string]any, k string) bool {
|
||||
switch v := a[k].(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
return strings.EqualFold(strings.TrimSpace(v), "true")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func intArg(a map[string]any, k string, otherwise int) int {
|
||||
if n, ok := a[k].(float64); ok && n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return otherwise
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its
|
||||
// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real
|
||||
// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from
|
||||
// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus,
|
||||
// writes them as findings into its bucket on the bus, keeps the verdict, and says checked.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
type wire struct {
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
|
||||
type player struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
mu sync.Mutex
|
||||
in io.Writer
|
||||
next int
|
||||
pending map[string]chan wire
|
||||
}
|
||||
|
||||
func (p *player) send(m map[string]any) {
|
||||
raw, _ := json.Marshal(m)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
_, _ = p.in.Write(append(raw, '\n'))
|
||||
}
|
||||
|
||||
func (p *player) answer(id json.RawMessage, result any, err error) {
|
||||
if err != nil {
|
||||
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
p.send(map[string]any{"id": id, "result": result})
|
||||
}
|
||||
|
||||
func (p *player) serve(out io.Reader) {
|
||||
scanner := bufio.NewScanner(out)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
for scanner.Scan() {
|
||||
var m wire
|
||||
if json.Unmarshal(scanner.Bytes(), &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.Method == "" {
|
||||
var key string
|
||||
_ = json.Unmarshal(m.ID, &key)
|
||||
p.mu.Lock()
|
||||
ch := p.pending[key]
|
||||
delete(p.pending, key)
|
||||
p.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- m
|
||||
}
|
||||
continue
|
||||
}
|
||||
go p.handle(m)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) kv(state string) (jetstream.KeyValue, error) {
|
||||
return p.js.KeyValue(p.ctx, Module+"_"+state)
|
||||
}
|
||||
|
||||
func (p *player) handle(m wire) {
|
||||
var a struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
switch m.Method {
|
||||
case "mesh/state.get":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
e, err := kv.Get(p.ctx, a.Key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
p.answer(m.ID, nil, nil)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
|
||||
case "mesh/state.put":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
var rev uint64
|
||||
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, nil, err)
|
||||
case "mesh/state.keys":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
keys, err := kv.Keys(p.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
keys, err = []string{}, nil
|
||||
}
|
||||
p.answer(m.ID, keys, err)
|
||||
case "mesh/state.delete":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
err = kv.Delete(p.ctx, a.Key)
|
||||
}
|
||||
p.answer(m.ID, map[string]any{}, err)
|
||||
case "mesh/seat.publish":
|
||||
var e struct {
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
|
||||
case "mesh/publish":
|
||||
var e struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
|
||||
case "mesh/ask":
|
||||
var ask struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &ask)
|
||||
if ask.Key != StreamsTool {
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
return
|
||||
}
|
||||
// The bus module's answer, from the server's own streams.
|
||||
var detail []map[string]any
|
||||
for name := range p.js.StreamNames(p.ctx).Name() {
|
||||
detail = append(detail, map[string]any{"name": name})
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"detail": detail}, nil)
|
||||
default:
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("c-%d", p.next)
|
||||
ch := make(chan wire, 1)
|
||||
p.pending[id] = ch
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
|
||||
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
|
||||
select {
|
||||
case m := <-ch:
|
||||
if m.Error != nil {
|
||||
return nil, errors.New(m.Error.Message)
|
||||
}
|
||||
var r struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
|
||||
return nil, fmt.Errorf("%s answered %s", name, m.Result)
|
||||
}
|
||||
if r.IsError {
|
||||
return nil, errors.New(r.Content[0].Text)
|
||||
}
|
||||
return json.RawMessage(r.Content[0].Text), nil
|
||||
case <-time.After(60 * time.Second):
|
||||
return nil, fmt.Errorf("%s did not answer in 60s", name)
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus.
|
||||
func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, Module)
|
||||
build := exec.Command("go", "build", "-o", bin, ".")
|
||||
build.Stderr = os.Stderr
|
||||
if err := build.Run(); err != nil {
|
||||
t.Fatalf("the self-check does not build: %v", err)
|
||||
}
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{FindingsState, VerdictState, VerdictsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin)
|
||||
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
|
||||
stdin, _ := cmd.StdinPipe()
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
|
||||
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
|
||||
go p.serve(stdout)
|
||||
|
||||
raw, err := p.call(DoctorTool, map[string]any{"run": true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var answer struct {
|
||||
Run Run `json:"run"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verdicts := map[string]string{}
|
||||
for _, v := range answer.Run.Probes {
|
||||
verdicts[v.ID] = v.Verdict
|
||||
}
|
||||
if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun ||
|
||||
verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") {
|
||||
t.Errorf("the run answered %+v", answer.Run)
|
||||
}
|
||||
msg, err := checked.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("checked was not said: %v", err)
|
||||
}
|
||||
var said Run
|
||||
if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run {
|
||||
t.Errorf("checked said %s", msg.Data)
|
||||
}
|
||||
findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState)
|
||||
e, err := findings.Get(ctx, "bus.EVENTS.stream-missing")
|
||||
if err != nil {
|
||||
t.Fatalf("the missing stream is not a finding on the bus: %v", err)
|
||||
}
|
||||
f, err := health.DecodeFinding(e.Value())
|
||||
if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
|
||||
t.Errorf("the finding is %s", e.Value())
|
||||
}
|
||||
verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState)
|
||||
if _, err := verdict.Get(ctx, LastKey); err != nil {
|
||||
t.Errorf("the last verdict is not kept: %v", err)
|
||||
}
|
||||
if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) {
|
||||
t.Errorf("the week's verdicts answered %s, %v", raw, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user