Say when the mesh is wrong: conditions, watchdogs, the bus's advisories, doctor (hq to-be 45 Phase 1)
Every one of the 48 core failures of research 031 was found by a person looking; the mesh's answers carried the fact for whoever asked and told nobody. - The condition store (to-be 45 §2): mesh-controller_conditions, one key per open condition, written by compare-and-set so a person's silence and the watchdogs never lose each other's word; every transition kept ninety days in mesh-controller_condition-history and said as the seat's events condition-raised / condition-changed / condition-cleared (the condition at the top level, with event, at, change, why, show), offered again while the bus is away. Raised and cleared by observation only; a clearing reopened within ten minutes is the same condition with its count up, its silence kept. Verbs: conditions, conditions show, conditions silence (a hand act, at most a week), conditions history. - ADR 0224's provider standing is the first kind, provider-failing, held by the provider's events; the provider_standing table is no longer read or written (left in place: dropping it is the operator's word). - status leads with the open conditions, urgent first, and says all well only with none open; conditions it cannot read are said and not well. - The signals table compiled in, one watchdog loop over it every 30s: S1 heartbeat (3 intervals, asleep machines excepted, control node urgent after 30 min), S2 report after a send, S3 plan tier, S4 event loop deaf, S5 merge not acted, S6 ask lost, S7 call hung, S8 provider silent, S9 advisories, S10 self-check silent, S11 node tools silent, S13 stale refusals; S12, S14, S15 deferred with their reasons. A row that cannot see raises probe-failed and clears nothing. A test generated from the table suppresses each signal inside and past its bound. - The bus's advisories (maximum deliveries, a mesh consumer deleted) and the controller's own slow consumer and refused subjects, said in the mesh's words. - doctor: the probe registry D1-D10 (D5 deferred) and DW, every five minutes, each in thirty seconds; a probe that cannot run is never a pass. D1 validates with mesh-host's own validator. Every run ends with the doctor-heartbeat event mesh-watcher listens for. - The controller is granted its new buckets, events, the two advisories and $SRV.INFO; the node tools their tools-alive heartbeat. The streams and consumers the controller asserts and the ones D6/D7 expect are one derivation.
This commit is contained in:
jochen
@@ -0,0 +1,764 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"github.com/novox/mesh-host/validate"
|
||||
"golang.org/x/net/dns/dnsmessage"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
||||
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||
// look" (ADR 0227 rule 4).
|
||||
|
||||
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
open := d.open
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
// The private network every declaration is composed with. Not computable is not "could not
|
||||
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
||||
// do not resolve, which are usually why, are named below.
|
||||
gens, gensErr := generators(ctx, open)
|
||||
if gensErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
||||
Token: "uncomposable", Severity: conditions.Urgent,
|
||||
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
||||
Said: firstLine(gensErr.Error())})
|
||||
}
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil && !unresolvable(err) {
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems []string
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||
var body []byte
|
||||
if body, err = declared.Body(); err == nil {
|
||||
problems = validate.Declaration(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
||||
oneLine(err.Error())),
|
||||
Said: oneLine(err.Error())})
|
||||
case len(problems) > 0:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
||||
Machine: n.Name, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||
n.Name, len(problems), problems[0]),
|
||||
Said: strings.Join(problems, "; ")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||
// takes as final (issue 262).
|
||||
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
holders, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvers := holders["mesh-dns-resolver"]
|
||||
if len(resolvers) == 0 {
|
||||
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suffix := overlay.Suffix()
|
||||
var out []conditions.Observation
|
||||
for holder, at := range resolvers {
|
||||
var wrong []string
|
||||
for _, p := range places {
|
||||
if p.Address == "" {
|
||||
continue
|
||||
}
|
||||
name := p.Name + "." + suffix
|
||||
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
||||
switch {
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
||||
continue
|
||||
case rcode != dnsmessage.RCodeSuccess:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
||||
case !slices.Contains(v4, p.Address):
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
||||
}
|
||||
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
||||
switch {
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
||||
case rcode != dnsmessage.RCodeSuccess:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
||||
"takes that as no such name", holder, rcode, name))
|
||||
case len(v6) > 0:
|
||||
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
||||
}
|
||||
}
|
||||
if len(wrong) > 0 {
|
||||
node := strings.TrimSuffix(holder, "."+suffix)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
||||
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
||||
node, wrong[0]),
|
||||
Said: strings.Join(wrong, "; ")})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
||||
var resolverPort = "53"
|
||||
|
||||
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
||||
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
||||
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
||||
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
||||
packed, err := msg.Pack()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
dialer := net.Dialer{Timeout: 3 * time.Second}
|
||||
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
if _, err := conn.Write(packed); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
buf := make([]byte, 1500)
|
||||
n, err := conn.Read(buf)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
||||
}
|
||||
var answer dnsmessage.Message
|
||||
if err := answer.Unpack(buf[:n]); err != nil {
|
||||
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
||||
}
|
||||
var addresses []string
|
||||
for _, a := range answer.Answers {
|
||||
switch r := a.Body.(type) {
|
||||
case *dnsmessage.AResource:
|
||||
addresses = append(addresses, net.IP(r.A[:]).String())
|
||||
case *dnsmessage.AAAAResource:
|
||||
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
||||
}
|
||||
}
|
||||
return addresses, answer.RCode, nil
|
||||
}
|
||||
|
||||
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
||||
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
||||
// S1's, and is not asked about here.
|
||||
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recorded, err := d.open.inventory.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
expected[seat] = map[string]bool{}
|
||||
}
|
||||
expected[seat][node] = true
|
||||
}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
||||
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
||||
}
|
||||
onRecord := false
|
||||
for _, h := range recorded {
|
||||
if h.Claim == s.Name && heard[h.Node] {
|
||||
add(s.Name, h.Node)
|
||||
onRecord = true
|
||||
}
|
||||
}
|
||||
if onRecord && s.Scope == catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name != s.Name {
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(expected) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
answering, err := discoverHolders(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for seat, nodes := range expected {
|
||||
for node := range nodes {
|
||||
if answering[seat][node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
||||
Token: "silent", Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
||||
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||
func heardMachines(d *doctor) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if d.watchdogs == nil {
|
||||
return out
|
||||
}
|
||||
f := d.watchdogs.lastFacts()
|
||||
if f == nil {
|
||||
return out
|
||||
}
|
||||
for _, m := range f.machines {
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
||||
// the most (ADR 0197: a large runtime's answer arrives last).
|
||||
const (
|
||||
discoveryQuiet = 1500 * time.Millisecond
|
||||
discoveryPatience = 8 * time.Second
|
||||
)
|
||||
|
||||
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
||||
// endpoint a seat's verb is served on.
|
||||
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
||||
inbox := conn.NewRespInbox()
|
||||
sub, err := conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
out := map[string]map[string]bool{}
|
||||
deadline := time.Now().Add(discoveryPatience)
|
||||
for time.Now().Before(deadline) {
|
||||
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
||||
if seat == "" {
|
||||
continue
|
||||
}
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
if out[seat] == nil {
|
||||
out[seat] = map[string]bool{}
|
||||
}
|
||||
out[seat][node] = true
|
||||
}
|
||||
// A holder that announces itself as its seat, by name and machine.
|
||||
if out[info.Name] == nil {
|
||||
out[info.Name] = map[string]bool{}
|
||||
}
|
||||
out[info.Name][info.ID] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
||||
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if store == "" {
|
||||
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
||||
}
|
||||
var report collectionReport
|
||||
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
||||
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
if n := len(report.Unheld); n > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
||||
"delete them", n, len(kept)),
|
||||
Said: "first: " + report.Unheld[0]})
|
||||
}
|
||||
if n := len(report.Missing); n > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
||||
Kind: "archives-missing", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
||||
Said: "first: " + report.Missing[0]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
||||
const consumerFarBehind = 1000
|
||||
|
||||
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
||||
// and is near its stream's head.
|
||||
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, c := range consumers {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
||||
who := c.Stream + "." + c.Name
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
||||
Kind: "consumer-lost", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
||||
Said: "no consumer " + c.Name + " on " + c.Stream})
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
||||
}
|
||||
if differs := consumerDiffers(c, info.Config); differs != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
||||
Said: differs})
|
||||
}
|
||||
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
||||
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// consumerWords is a consumer as the mesh says it, with its name.
|
||||
func consumerWords(c broker.Consumer) string {
|
||||
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
||||
}
|
||||
|
||||
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
||||
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
||||
// stated choice (novox/hq issue 156).
|
||||
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
||||
var differs []string
|
||||
haveFilters := append([]string(nil), have.FilterSubjects...)
|
||||
if have.FilterSubject != "" {
|
||||
haveFilters = append(haveFilters, have.FilterSubject)
|
||||
}
|
||||
wantFilters := append([]string(nil), want.Filters...)
|
||||
sort.Strings(haveFilters)
|
||||
sort.Strings(wantFilters)
|
||||
if !slices.Equal(haveFilters, wantFilters) {
|
||||
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||
}
|
||||
if have.AckPolicy != nats.AckExplicitPolicy {
|
||||
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
||||
}
|
||||
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
||||
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
||||
}
|
||||
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
||||
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
||||
}
|
||||
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
||||
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
||||
}
|
||||
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
||||
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
||||
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := d.js.Context()
|
||||
var out []conditions.Observation
|
||||
for _, s := range streams {
|
||||
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrStreamNotFound):
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
||||
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||
Said: "no stream " + s.Name})
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
||||
}
|
||||
if differs := streamDiffers(s, info.Config); differs != "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
||||
Kind: "stream-wrong", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
||||
Said: differs})
|
||||
}
|
||||
}
|
||||
for _, bucket := range broker.ControllerBuckets() {
|
||||
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
||||
Kind: "stream-wrong", Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
||||
Said: "no bucket " + bucket})
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
||||
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
||||
var differs []string
|
||||
haveSubjects := append([]string(nil), have.Subjects...)
|
||||
wantSubjects := append([]string(nil), want.Subjects...)
|
||||
sort.Strings(haveSubjects)
|
||||
sort.Strings(wantSubjects)
|
||||
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||
}
|
||||
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
||||
switch want.Retention {
|
||||
case broker.RetentionWorkQueue:
|
||||
retention = nats.WorkQueuePolicy
|
||||
case broker.RetentionLastPerSubject:
|
||||
perSubject = 1
|
||||
}
|
||||
if have.Retention != retention {
|
||||
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||
}
|
||||
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
||||
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||
|
||||
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
||||
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
||||
// machine's holder of `node-intrusion-prevention` that is heard from.
|
||||
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owned := map[string]string{} // address → whose
|
||||
for _, p := range places {
|
||||
if p.Address != "" {
|
||||
owned[p.Address] = p.Name + "'s private address"
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
owned[ip.String()] = p.Name + "'s endpoint"
|
||||
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
||||
for _, a := range addrs {
|
||||
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var holders []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if c.Name == "node-intrusion-prevention" {
|
||||
for _, node := range e.On {
|
||||
if heard[node] && !slices.Contains(holders, node) {
|
||||
holders = append(holders, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(holders)
|
||||
var out []conditions.Observation
|
||||
var unasked []string
|
||||
for _, node := range holders {
|
||||
answer, err := askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
||||
if err != nil {
|
||||
unasked = append(unasked, err.Error())
|
||||
continue
|
||||
}
|
||||
var banned []string
|
||||
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
||||
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
||||
banned = append(banned, ip+" ("+whose+")")
|
||||
}
|
||||
}
|
||||
if len(banned) > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
||||
Machine: node, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
||||
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
||||
Said: strings.Join(banned, ", ")})
|
||||
}
|
||||
}
|
||||
if len(unasked) > 0 {
|
||||
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
||||
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
started := time.Now()
|
||||
stale := ""
|
||||
if statusFrom != nil {
|
||||
answer, err := statusFrom.answer(ctx)
|
||||
if err != nil {
|
||||
stale = err.Error()
|
||||
} else if m, ok := answer.(map[string]any); ok {
|
||||
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
||||
stale = "its last composition failed: " + failed
|
||||
}
|
||||
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
||||
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
||||
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
||||
}
|
||||
}
|
||||
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
||||
stale = err.Error()
|
||||
}
|
||||
took := time.Since(started)
|
||||
var out []conditions.Observation
|
||||
if took > 10*time.Second || stale != "" {
|
||||
why := stale
|
||||
if why == "" {
|
||||
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
||||
Machine: d.host, Severity: conditions.Warning,
|
||||
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
||||
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
||||
// the build the machine was last sent, once it reported applying that send.
|
||||
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := map[string]bool{}
|
||||
for _, r := range reports {
|
||||
applied[r.Node] = r.Current
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
var out []conditions.Observation
|
||||
for _, n := range nodes {
|
||||
if !heard[n.Name] {
|
||||
continue // a machine not heard from is S1's
|
||||
}
|
||||
var behind []string
|
||||
host := current[hostModule].Commit
|
||||
if host != "" && n.HostVersion != "" && !rolling[hostModule] && !sameCommit(n.HostVersion, host) {
|
||||
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s", short(n.HostVersion), short(host)))
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tools := current[broker.RuntimeModule].Commit
|
||||
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
case !sameCommit(sent[broker.RuntimeModule], tools):
|
||||
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
||||
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
||||
case !applied[n.Name]:
|
||||
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
||||
Machine: n.Name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
||||
Said: strings.Join(behind, "; ")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hostModule is the node-engine's module.
|
||||
const hostModule = "mesh-host"
|
||||
|
||||
// sameCommit says two commits are one, either written short.
|
||||
func sameCommit(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||
}
|
||||
|
||||
func orNotKnown(s string) string {
|
||||
if s == "" {
|
||||
return "(not known)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
||||
// watchdogs watch each other: S10 is the other half.
|
||||
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.watchdogs == nil {
|
||||
return nil, nil // a self-check run outside the serving controller has none to watch
|
||||
}
|
||||
ticked := d.watchdogs.lastTick()
|
||||
since := ticked
|
||||
if since.IsZero() {
|
||||
since = d.watchdogs.started
|
||||
}
|
||||
if time.Since(since) <= 3*watchEvery {
|
||||
return nil, nil
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
||||
Machine: d.host, Severity: conditions.Urgent,
|
||||
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
||||
since.UTC().Format("2006-01-02 15:04 MST")),
|
||||
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
||||
}
|
||||
|
||||
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
||||
// own refusal is an error.
|
||||
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
||||
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
||||
}
|
||||
return answer.Result, nil
|
||||
}
|
||||
|
||||
// oneLine is a message of several lines said on one, its runs of space made one.
|
||||
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
Reference in New Issue
Block a user