Archived
Research 031 counted the repairs people made by hand: a push to unstick a plan waiting on a report, a controller restarted to make an object again, a plan closed, a consumer re-made from now. Each was the ordinary path taken again by someone who noticed. The healer registry makes each a registered response to one condition kind, with a budget, a settle and its event: - H1 sent-not-reported: ask the machine's node-engine to report again (mesh.node.<n>.ask.report); if it does not report what it was sent, send it again, never moving a build a policy or a plan holds back - H2 stalled: close a plan whose wait is superseded or finished - H3 holder-silent / consumer-lost: the send's own assertion of the bus's objects (issue 208's note) - H4 consumer-behind: consumer-reset, only for a consumer the stream table marks resettable (the controller's own events consumer) - H5 is the identity provider's own repair (ADR 0224 §5), registered only Success is the observation clearing the condition, never the healer; a spent budget hands the condition to the operator, urgent, with what was tried, and no healer touches it again. Every act is begun in the store before it is made (migration 0070), kept in the condition's tried as "healer Hn" and said as the seat event healer-acted; a heal is never a hand act. More than twelve acts in an hour stop every healer until an hour after the last, said urgently. Only the lease holder heals. S15 is live: a cause repaired by hand twice in a fortnight raises healer-wanted, naming the healer that was not enough where one exists. D6's far-behind finding has its own kind, consumer-behind. Nodes are granted the question; the controller's grant gains healer-acted (genesis lock in mesh-host). `healers` lists the registry, the acts and the brake; status counts the week's heals.
904 lines
34 KiB
Go
904 lines
34 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
"github.com/nats-io/nats.go/micro"
|
|
"github.com/novox/mesh-host/validate"
|
|
"golang.org/x/net/dns/dnsmessage"
|
|
|
|
"github.com/novox/mesh-controller/internal/artifacts"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
"github.com/novox/mesh-controller/internal/overlay"
|
|
)
|
|
|
|
// The probes of the self-check's registry (doctor.go), one function each. A probe answers what is
|
|
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
|
// look" (ADR 0227 rule 4).
|
|
|
|
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
|
// (mesh-host's `validate`, the package the host runs) takes it.
|
|
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
open := d.open
|
|
nodes, err := open.inventory.Nodes(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []conditions.Observation
|
|
// The private network every declaration is composed with. Not computable is not "could not
|
|
// look": it is the finding — no machine's declaration composes without it — and the machines that
|
|
// do not resolve, which are usually why, are named below.
|
|
gens, gensErr := generators(ctx, open)
|
|
if gensErr != nil {
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "private-network",
|
|
Token: "uncomposable", Severity: conditions.Urgent,
|
|
Summary: "the private network cannot be computed, so no machine's declaration composes",
|
|
Said: firstLine(gensErr.Error())})
|
|
}
|
|
for _, n := range nodes {
|
|
plan, settings, err := planFor(ctx, open, n.Name)
|
|
if err != nil && !unresolvable(err) {
|
|
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
|
}
|
|
var problems []string
|
|
if err == nil && gensErr == nil {
|
|
var declared sendable
|
|
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
|
// With the order it was last sent, so the validator reads the envelope a machine is sent
|
|
// — its epoch included (novox/hq to-be 45 §6).
|
|
var serr error
|
|
if declared.Sequence, serr = open.inventory.Sequence(ctx, n.ID); serr == nil {
|
|
declared.Epoch, serr = open.inventory.SentEpoch(ctx, n.ID)
|
|
}
|
|
if serr != nil {
|
|
return nil, serr // the store, not the machine: the probe could not run
|
|
}
|
|
var body []byte
|
|
if body, err = declared.Body(); err == nil {
|
|
problems = validate.Declaration(body)
|
|
}
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
switch {
|
|
case err != nil:
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "uncomposable",
|
|
Machine: n.Name, Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("nothing can be sent to %s: its declaration does not compose — %s", n.Name,
|
|
oneLine(err.Error())),
|
|
Said: oneLine(err.Error())})
|
|
case len(problems) > 0:
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "refused",
|
|
Machine: n.Name, Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
|
n.Name, len(problems), problems[0]),
|
|
Said: strings.Join(problems, "; ")})
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
|
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
|
// takes as final (issue 262).
|
|
func probeResolvers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
inv := d.open.inventory
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
holders, err := replicatedHolders(ctx, inv, shelf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resolvers := holders["mesh-dns-resolver"]
|
|
if len(resolvers) == 0 {
|
|
return nil, nil // the mesh holds no resolver of its own: nothing is asked of one
|
|
}
|
|
places, err := onTheNetwork(ctx, inv, shelf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
suffix := overlay.Suffix()
|
|
var out []conditions.Observation
|
|
for holder, at := range resolvers {
|
|
var wrong []string
|
|
for _, p := range places {
|
|
if p.Address == "" {
|
|
continue
|
|
}
|
|
name := p.Name + "." + suffix
|
|
v4, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeA)
|
|
switch {
|
|
case err != nil:
|
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "A", name, err))
|
|
continue
|
|
case rcode != dnsmessage.RCodeSuccess:
|
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (A)", holder, rcode, name))
|
|
case !slices.Contains(v4, p.Address):
|
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (A), not %s", holder, v4, name, p.Address))
|
|
}
|
|
v6, rcode, err := askResolver(ctx, at, name, dnsmessage.TypeAAAA)
|
|
switch {
|
|
case err != nil:
|
|
wrong = append(wrong, fmt.Sprintf("%s for %s: %v", "AAAA", name, err))
|
|
case rcode != dnsmessage.RCodeSuccess:
|
|
wrong = append(wrong, fmt.Sprintf("%s answered %s for %s (AAAA), not NODATA: a musl machine "+
|
|
"takes that as no such name", holder, rcode, name))
|
|
case len(v6) > 0:
|
|
wrong = append(wrong, fmt.Sprintf("%s answered %v for %s (AAAA); the mesh has no IPv6 addresses", holder, v6, name))
|
|
}
|
|
}
|
|
if len(wrong) > 0 {
|
|
node := strings.TrimSuffix(holder, "."+suffix)
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver." + node,
|
|
Token: "wrong", Machine: node, Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s",
|
|
node, wrong[0]),
|
|
Said: strings.Join(wrong, "; ")})
|
|
}
|
|
}
|
|
return sortedFound(out), nil
|
|
}
|
|
|
|
// resolverPort is where a resolver answers; a variable so a test can stand one up.
|
|
var resolverPort = "53"
|
|
|
|
// askResolver asks one resolver one question over UDP, and answers the addresses and the code.
|
|
func askResolver(ctx context.Context, at, name string, kind dnsmessage.Type) ([]string, dnsmessage.RCode, error) {
|
|
q, err := dnsmessage.NewName(strings.TrimSuffix(name, ".") + ".")
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
msg := dnsmessage.Message{Header: dnsmessage.Header{ID: uint16(time.Now().UnixNano()), RecursionDesired: true},
|
|
Questions: []dnsmessage.Question{{Name: q, Type: kind, Class: dnsmessage.ClassINET}}}
|
|
packed, err := msg.Pack()
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
dialer := net.Dialer{Timeout: 3 * time.Second}
|
|
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(at, resolverPort))
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
defer conn.Close()
|
|
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
|
if _, err := conn.Write(packed); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
buf := make([]byte, 1500)
|
|
n, err := conn.Read(buf)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("no answer from %s: %w", at, err)
|
|
}
|
|
var answer dnsmessage.Message
|
|
if err := answer.Unpack(buf[:n]); err != nil {
|
|
return nil, 0, fmt.Errorf("an answer from %s that cannot be read: %w", at, err)
|
|
}
|
|
var addresses []string
|
|
for _, a := range answer.Answers {
|
|
switch r := a.Body.(type) {
|
|
case *dnsmessage.AResource:
|
|
addresses = append(addresses, net.IP(r.A[:]).String())
|
|
case *dnsmessage.AAAAResource:
|
|
addresses = append(addresses, net.IP(r.AAAA[:]).String())
|
|
}
|
|
}
|
|
return addresses, answer.RCode, nil
|
|
}
|
|
|
|
// probeHolders is D3: every seat that serves verbs has, on every machine that holds it and is heard
|
|
// from, a holder answering the bus's discovery for that seat. A machine past its heartbeat's bound is
|
|
// S1's, and is not asked about here.
|
|
func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
entries, err := d.open.inventory.Catalogued(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
recorded, err := d.open.inventory.Holdings(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
heard := heardMachines(d)
|
|
expected := map[string]map[string]bool{} // seat → machine
|
|
add := func(seat, node string) {
|
|
if expected[seat] == nil {
|
|
expected[seat] = map[string]bool{}
|
|
}
|
|
expected[seat][node] = true
|
|
}
|
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
|
if len(s.Serves) == 0 || s.Name == catalogue.ControllerSeatName {
|
|
continue // a seat with no verb has nothing to answer with; this controller is answering now
|
|
}
|
|
onRecord := false
|
|
for _, h := range recorded {
|
|
if h.Claim == s.Name && heard[h.Node] {
|
|
add(s.Name, h.Node)
|
|
onRecord = true
|
|
}
|
|
}
|
|
if onRecord && s.Scope == catalogue.ScopeMesh {
|
|
continue
|
|
}
|
|
for _, e := range entries {
|
|
for _, c := range e.Manifest.Claims {
|
|
if c.Name != s.Name {
|
|
continue
|
|
}
|
|
for _, node := range e.On {
|
|
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
|
add(s.Name, node)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(expected) == 0 {
|
|
return nil, nil
|
|
}
|
|
answering, err := discoverHolders(ctx, d.js.Conn())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []conditions.Observation
|
|
for seat, nodes := range expected {
|
|
for node := range nodes {
|
|
if answering[seat][node] {
|
|
continue
|
|
}
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeSeat, ID: seat + "." + node,
|
|
Token: "silent", Machine: node, Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%s's holder on %s does not answer the bus: its verbs reach nothing there", seat, node),
|
|
Said: fmt.Sprintf("no answer for %s from %s to the bus's discovery", seat, node)})
|
|
}
|
|
}
|
|
return sortedFound(out), nil
|
|
}
|
|
|
|
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
|
func heardMachines(d *doctor) map[string]bool {
|
|
out := map[string]bool{}
|
|
if d.watchdogs == nil {
|
|
return out
|
|
}
|
|
f := d.watchdogs.lastFacts()
|
|
if f == nil {
|
|
return out
|
|
}
|
|
for _, m := range f.machines {
|
|
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
|
out[m.name] = true
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// discoveryPatience is how long the discovery's answers are waited for after the last arrived, and at
|
|
// the most (ADR 0197: a large runtime's answer arrives last).
|
|
const (
|
|
discoveryQuiet = 1500 * time.Millisecond
|
|
discoveryPatience = 8 * time.Second
|
|
)
|
|
|
|
// discoverHolders asks the bus's discovery who serves what, and answers seat → machine for every
|
|
// endpoint a seat's verb is served on.
|
|
func discoverHolders(ctx context.Context, conn *nats.Conn) (map[string]map[string]bool, error) {
|
|
inbox := conn.NewRespInbox()
|
|
sub, err := conn.SubscribeSync(inbox)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = sub.Unsubscribe() }()
|
|
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
|
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
|
}
|
|
out := map[string]map[string]bool{}
|
|
deadline := time.Now().Add(discoveryPatience)
|
|
for time.Now().Before(deadline) {
|
|
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
|
msg, err := sub.NextMsgWithContext(wait)
|
|
cancel()
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
break
|
|
}
|
|
var info micro.Info
|
|
if json.Unmarshal(msg.Data, &info) != nil {
|
|
continue
|
|
}
|
|
for _, e := range info.Endpoints {
|
|
seat, node := e.Metadata["seat"], e.Metadata["node"]
|
|
if seat == "" {
|
|
continue
|
|
}
|
|
if node == "" {
|
|
node = info.ID
|
|
}
|
|
if out[seat] == nil {
|
|
out[seat] = map[string]bool{}
|
|
}
|
|
out[seat][node] = true
|
|
}
|
|
// A holder that announces itself as its seat, by name and machine.
|
|
if out[info.Name] == nil {
|
|
out[info.Name] = map[string]bool{}
|
|
}
|
|
out[info.Name][info.ID] = true
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// probeArchives is D4: every archive the mesh keeps is held by its manifest in the artifact store.
|
|
func probeArchives(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
inv := d.open.inventory
|
|
kept, err := inv.KeptArchives(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(kept) == 0 {
|
|
return nil, nil
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
store, err := artifactStoreAddress(ctx, inv, shelf, "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if store == "" {
|
|
return nil, errors.New("the mesh keeps archives and has no artifact store on its network to ask")
|
|
}
|
|
var report collectionReport
|
|
if unasked, stopped := askHeld(ctx, artifacts.Store{Address: store}, kept, &report); stopped != "" {
|
|
return nil, fmt.Errorf("%d kept archive(s) could not be asked about: %s", unasked, stopped)
|
|
}
|
|
var out []conditions.Observation
|
|
if n := len(report.Unheld); n > 0 {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-unheld",
|
|
Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%d of %d kept archive(s) are not held by a manifest: the store's collector would "+
|
|
"delete them", n, len(kept)),
|
|
Said: "first: " + report.Unheld[0]})
|
|
}
|
|
if n := len(report.Missing); n > 0 {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMesh, ID: "artifact-store", Token: "archives-missing",
|
|
Kind: "archives-missing", Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%d kept archive(s) are not in the artifact store at all", n),
|
|
Said: "first: " + report.Missing[0]})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// consumerFarBehind is how far a durable consumer may be from its stream's head (D6).
|
|
const consumerFarBehind = 1000
|
|
|
|
// probeConsumers is D6: every durable consumer the mesh expects exists, as the controller defines it,
|
|
// and is near its stream's head.
|
|
func probeConsumers(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
_, consumers, err := expectedBusObjects(ctx, d.open.inventory)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
js := d.js.Context()
|
|
var out []conditions.Observation
|
|
for _, c := range consumers {
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
info, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx))
|
|
who := c.Stream + "." + c.Name
|
|
switch {
|
|
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "missing",
|
|
Kind: "consumer-lost", Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(c)),
|
|
Said: "no consumer " + c.Name + " on " + c.Stream})
|
|
continue
|
|
case err != nil:
|
|
return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err)
|
|
}
|
|
if differs := consumerDiffers(c, info.Config); differs != "" {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "redefined",
|
|
Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%s is not as the controller defines it: %s", consumerWords(c), differs),
|
|
Said: differs})
|
|
}
|
|
if c.Stream != "NODES" && info.NumPending > consumerFarBehind {
|
|
// Its own kind (novox/hq to-be 45 §7): healer H4 answers a consumer far behind, and only
|
|
// this — a redefined consumer is not one a reset repairs.
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind",
|
|
Kind: kindConsumerBehind, Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(c), info.NumPending),
|
|
Said: fmt.Sprintf("%d pending, %d handed out and not settled", info.NumPending, info.NumAckPending)})
|
|
}
|
|
}
|
|
return sortedFound(out), nil
|
|
}
|
|
|
|
// consumerWords is a consumer as the mesh says it, with its name.
|
|
func consumerWords(c broker.Consumer) string {
|
|
return fmt.Sprintf("%s (%s on %s)", link.ConsumerInWords(c.Stream, c.Name), c.Name, c.Stream)
|
|
}
|
|
|
|
// consumerDiffers is what about a consumer on the bus is not as defined; empty when nothing is. The
|
|
// delivery subject is not compared: one kept as it was while a holder is bound is the assertion's
|
|
// stated choice (novox/hq issue 156).
|
|
func consumerDiffers(want broker.Consumer, have nats.ConsumerConfig) string {
|
|
var differs []string
|
|
haveFilters := append([]string(nil), have.FilterSubjects...)
|
|
if have.FilterSubject != "" {
|
|
haveFilters = append(haveFilters, have.FilterSubject)
|
|
}
|
|
wantFilters := append([]string(nil), want.Filters...)
|
|
sort.Strings(haveFilters)
|
|
sort.Strings(wantFilters)
|
|
if !slices.Equal(haveFilters, wantFilters) {
|
|
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
|
}
|
|
if have.AckPolicy != nats.AckExplicitPolicy {
|
|
differs = append(differs, "acknowledges "+have.AckPolicy.String()+", defined explicit")
|
|
}
|
|
if wantMax := want.MaxDeliver; wantMax != 0 && have.MaxDeliver != wantMax {
|
|
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, wantMax))
|
|
}
|
|
if wantWait := time.Duration(want.AckWaitSeconds) * time.Second; wantWait != 0 && have.AckWait != wantWait {
|
|
differs = append(differs, fmt.Sprintf("waits %s for an acknowledgement, defined %s", have.AckWait, wantWait))
|
|
}
|
|
if want.MaxAckPending != 0 && have.MaxAckPending != want.MaxAckPending {
|
|
differs = append(differs, fmt.Sprintf("hands out %d at once, defined %d", have.MaxAckPending, want.MaxAckPending))
|
|
}
|
|
if wantPush, havePush := want.Push || want.Queue != "", have.DeliverSubject != ""; wantPush != havePush {
|
|
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
|
}
|
|
return strings.Join(differs, "; ")
|
|
}
|
|
|
|
// probeStreams is D7: every stream the controller defines exists as defined, and its own buckets.
|
|
func probeStreams(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
streams, _, err := expectedBusObjects(ctx, d.open.inventory)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
js := d.js.Context()
|
|
var out []conditions.Observation
|
|
for _, s := range streams {
|
|
info, err := js.StreamInfo(s.Name, nats.Context(ctx))
|
|
switch {
|
|
case errors.Is(err, nats.ErrStreamNotFound):
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-missing",
|
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
|
Said: "no stream " + s.Name})
|
|
continue
|
|
case err != nil:
|
|
return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err)
|
|
}
|
|
if differs := streamDiffers(s, info.Config); differs != "" {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: s.Name, Token: "stream-redefined",
|
|
Kind: "stream-wrong", Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("the stream %s is not as the controller defines it: %s", s.Name, differs),
|
|
Said: differs})
|
|
}
|
|
}
|
|
for _, bucket := range broker.ControllerBuckets() {
|
|
if _, err := js.StreamInfo("KV_"+bucket, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: bucket, Token: "bucket-missing",
|
|
Kind: "stream-wrong", Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("the controller's bucket %s is not on the bus: what it keeps there is not kept", bucket),
|
|
Said: "no bucket " + bucket})
|
|
} else if err != nil {
|
|
return nil, fmt.Errorf("the bucket %s cannot be read: %w", bucket, err)
|
|
}
|
|
}
|
|
return sortedFound(out), nil
|
|
}
|
|
|
|
// streamDiffers is what about a stream on the bus is not as defined; empty when nothing is.
|
|
func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
|
|
var differs []string
|
|
haveSubjects := append([]string(nil), have.Subjects...)
|
|
wantSubjects := append([]string(nil), want.Subjects...)
|
|
sort.Strings(haveSubjects)
|
|
sort.Strings(wantSubjects)
|
|
if !slices.Equal(haveSubjects, wantSubjects) {
|
|
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
|
}
|
|
retention, perSubject := nats.LimitsPolicy, int64(want.MaxMsgsPerSubject)
|
|
switch want.Retention {
|
|
case broker.RetentionWorkQueue:
|
|
retention = nats.WorkQueuePolicy
|
|
case broker.RetentionLastPerSubject:
|
|
perSubject = 1
|
|
}
|
|
if have.Retention != retention {
|
|
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
|
}
|
|
if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
|
|
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
|
|
}
|
|
return strings.Join(differs, "; ")
|
|
}
|
|
|
|
// ipLike finds the addresses in a ban list, whatever shape its holder answers in.
|
|
var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
|
|
|
// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are
|
|
// every machine's private address and the address its endpoint names; the ban lists are asked of each
|
|
// machine's holder of `node-intrusion-prevention` that is heard from.
|
|
func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
inv := d.open.inventory
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
owned := map[string]string{} // address → whose
|
|
for _, p := range places {
|
|
if p.Address != "" {
|
|
owned[p.Address] = p.Name + "'s private address"
|
|
}
|
|
if host, _, err := net.SplitHostPort(p.Endpoint); err == nil && host != "" {
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
owned[ip.String()] = p.Name + "'s endpoint"
|
|
} else if addrs, err := net.DefaultResolver.LookupHost(ctx, host); err == nil {
|
|
for _, a := range addrs {
|
|
owned[a] = p.Name + "'s endpoint (" + host + ")"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
entries, err := inv.Catalogued(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
heard := heardMachines(d)
|
|
var holders []string
|
|
for _, e := range entries {
|
|
for _, c := range e.Manifest.Claims {
|
|
if c.Name == "node-intrusion-prevention" {
|
|
for _, node := range e.On {
|
|
if heard[node] && !slices.Contains(holders, node) {
|
|
holders = append(holders, node)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(holders)
|
|
var out []conditions.Observation
|
|
// Every machine asked at once: one after another, four holders that each wait their bound
|
|
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
|
|
answers := make([]json.RawMessage, len(holders))
|
|
errs := make([]error, len(holders))
|
|
var wg sync.WaitGroup
|
|
for i, node := range holders {
|
|
wg.Add(1)
|
|
go func(i int, node string) {
|
|
defer wg.Done()
|
|
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
|
|
}(i, node)
|
|
}
|
|
wg.Wait()
|
|
var unasked []string
|
|
for i, node := range holders {
|
|
answer, err := answers[i], errs[i]
|
|
if err != nil {
|
|
unasked = append(unasked, err.Error())
|
|
continue
|
|
}
|
|
var banned []string
|
|
for _, ip := range ipLike.FindAllString(string(answer), -1) {
|
|
if whose, ours := owned[ip]; ours && !slices.Contains(banned, ip+" ("+whose+")") {
|
|
banned = append(banned, ip+" ("+whose+")")
|
|
}
|
|
}
|
|
if len(banned) > 0 {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "bans-the-mesh",
|
|
Machine: node, Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es): %s — the mesh is locked out "+
|
|
"of itself there (ADR 0186)", node, len(banned), strings.Join(banned, ", ")),
|
|
Said: strings.Join(banned, ", ")})
|
|
}
|
|
}
|
|
if len(unasked) > 0 {
|
|
return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; "))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// probeStatus is D9: `status` answers in full within ten seconds, from a summary composed lately.
|
|
func probeStatus(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
started := time.Now()
|
|
stale := ""
|
|
if statusFrom != nil {
|
|
answer, err := statusFrom.answer(ctx)
|
|
if err != nil {
|
|
stale = err.Error()
|
|
} else if m, ok := answer.(map[string]any); ok {
|
|
if failed, _ := m["lastAttemptFailed"].(string); failed != "" {
|
|
stale = "its last composition failed: " + failed
|
|
}
|
|
if composed, err := time.Parse(time.RFC3339, fmt.Sprint(m["composed"])); err == nil &&
|
|
time.Since(composed) > 3*statusEvery+statusComposeWithin {
|
|
stale = fmt.Sprintf("it answers a summary composed %s ago", time.Since(composed).Round(time.Second))
|
|
}
|
|
}
|
|
} else if _, err := theThreeQuestions(ctx, d.open); err != nil {
|
|
stale = err.Error()
|
|
}
|
|
took := time.Since(started)
|
|
var out []conditions.Observation
|
|
if took > 10*time.Second || stale != "" {
|
|
why := stale
|
|
if why == "" {
|
|
why = fmt.Sprintf("it took %s", took.Round(time.Millisecond))
|
|
}
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller", Token: "status-slow",
|
|
Machine: d.host, Severity: conditions.Warning,
|
|
Summary: "status does not answer in full within ten seconds: " + why, Said: why})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// probeCoreBuilds is D10: every machine runs the node-engine and the node tools the mesh holds, or a
|
|
// plan is rolling one of them out. The node-engine says its build in each report; the node tools' is
|
|
// the build the machine was last sent, once it reported applying that send.
|
|
func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
inv := d.open.inventory
|
|
current, err := inv.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
plans, err := inv.OpenPlans(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hostVersions := deliveredVersions(shelf[hostModule])
|
|
rolling := map[string]bool{}
|
|
for _, p := range plans {
|
|
for m := range p.Modules {
|
|
rolling[m] = true
|
|
}
|
|
}
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reports, err := inv.LastReports(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
applied := map[string]bool{}
|
|
for _, r := range reports {
|
|
applied[r.Node] = r.Current
|
|
}
|
|
heard := heardMachines(d)
|
|
var out []conditions.Observation
|
|
for _, n := range nodes {
|
|
if !heard[n.Name] {
|
|
continue // a machine not heard from is S1's
|
|
}
|
|
var behind []string
|
|
// **A node-engine says its version as the directory it was delivered into** — its archive's
|
|
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
|
|
// from. Compared as a commit, every machine read as behind right after a push sent it the
|
|
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
|
|
// which a commit can match.
|
|
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
|
|
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
|
|
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
|
|
}
|
|
assigned, err := inv.Assigned(ctx, n.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tools := current[broker.RuntimeModule].Commit
|
|
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
|
|
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
switch {
|
|
case !known:
|
|
case !sameCommit(sent[broker.RuntimeModule], tools):
|
|
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
|
|
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
|
|
case !applied[n.Name]:
|
|
behind = append(behind, "the node tools it was last sent, not yet reported applied")
|
|
}
|
|
}
|
|
if len(behind) > 0 {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
|
|
Machine: n.Name, Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("%s runs %s, and no plan is rolling them out", n.Name, strings.Join(behind, "; ")),
|
|
Said: strings.Join(behind, "; ")})
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
|
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
|
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
|
func deliveredVersions(m catalogue.Manifest) []string {
|
|
var out []string
|
|
for _, r := range m.Resources {
|
|
path, _ := r["path"].(string)
|
|
before, version, found := strings.Cut(path, "/versions/")
|
|
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
|
|
continue
|
|
}
|
|
if !slices.Contains(out, version) {
|
|
out = append(out, version)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
|
|
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
|
|
// has not said, or a mesh that holds no delivered build, is not behind anything.
|
|
func engineBehind(reported string, delivered []string, commit string) bool {
|
|
if reported == "" || len(delivered) == 0 {
|
|
return false
|
|
}
|
|
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
|
|
}
|
|
|
|
// hostModule is the node-engine's module.
|
|
const hostModule = "mesh-host"
|
|
|
|
// sameCommit says two commits are one, either written short.
|
|
func sameCommit(a, b string) bool {
|
|
if a == "" || b == "" {
|
|
return false
|
|
}
|
|
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
|
}
|
|
|
|
func orNotKnown(s string) string {
|
|
if s == "" {
|
|
return "(not known)"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// probeWatchdogs is DW: the watchdogs ran within three of their intervals. The doctor and the
|
|
// watchdogs watch each other: S10 is the other half.
|
|
func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
if d.watchdogs == nil {
|
|
return nil, nil // a self-check run outside the serving controller has none to watch
|
|
}
|
|
ticked := d.watchdogs.lastTick()
|
|
since := ticked
|
|
if since.IsZero() {
|
|
since = d.watchdogs.started
|
|
}
|
|
if time.Since(since) <= 3*watchEvery {
|
|
return nil, nil
|
|
}
|
|
return []conditions.Observation{{Scope: conditions.ScopeCore, ID: "watchdogs", Token: "silent",
|
|
Machine: d.host, Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("the watchdogs of the signals table have not run since %s: no late signal is being said",
|
|
since.UTC().Format("2006-01-02 15:04 MST")),
|
|
Said: fmt.Sprintf("no tick for %s", time.Since(since).Round(time.Second))}}, nil
|
|
}
|
|
|
|
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
|
|
// own refusal is an error.
|
|
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
|
|
if who, declared := declaredBy(ctx, seat, verb); !declared {
|
|
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
|
|
"controller is not granted it", who, seat, verb)
|
|
}
|
|
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if answer.Error != "" {
|
|
return nil, fmt.Errorf("%s's %s.%s refused: %s", node, seat, verb, answer.Error)
|
|
}
|
|
return answer.Result, nil
|
|
}
|
|
|
|
// oneLine is a message of several lines said on one, its runs of space made one.
|
|
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
|
|
|
// probeLease is D5 (novox/hq to-be 45 §4, §6): exactly one lease holder — the key on the bus names this
|
|
// controller at the epoch it acts under, and the mesh's record has that epoch and no other open — and no
|
|
// message from a stale epoch refused in the last interval.
|
|
func probeLease(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
|
if d.js == nil {
|
|
return nil, errors.New("this controller is not on the bus")
|
|
}
|
|
st := theLease.standing()
|
|
var out []conditions.Observation
|
|
split := func(token, summary, said string) {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: token,
|
|
Machine: d.host, Severity: conditions.Urgent, Summary: summary, Said: said})
|
|
}
|
|
if st.Unleased != "" {
|
|
split("unheld", "this controller acts without the lease, so nothing keeps another from acting beside it: "+
|
|
st.Unleased, st.Unleased)
|
|
return out, nil
|
|
}
|
|
api, err := jetstream.New(d.js.Conn())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kv, err := api.KeyValue(ctx, broker.LeaseBucket)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the lease bucket cannot be read: %w", err)
|
|
}
|
|
holder, found, err := lease.Current(ctx, kv)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the lease cannot be read: %w", err)
|
|
}
|
|
switch {
|
|
case !found:
|
|
split("split", fmt.Sprintf("nobody holds the lease on the bus, and this controller acts as epoch %d", st.Epoch),
|
|
"the lease's key is absent")
|
|
case holder.Instance != instance || holder.Epoch != st.Epoch:
|
|
split("split", fmt.Sprintf("the lease on the bus names %s at epoch %d, and this controller (%s) acts as "+
|
|
"epoch %d: two controllers believe they may act", holder.Instance, holder.Epoch, instance, st.Epoch),
|
|
fmt.Sprintf("held by %s, epoch %d", holder.Instance, holder.Epoch))
|
|
}
|
|
// The record: one epoch open, this one.
|
|
epochs, err := d.open.inventory.EpochsSince(ctx, time.Now().Add(-time.Hour))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the epochs the mesh issued cannot be read: %w", err)
|
|
}
|
|
var open []string
|
|
for _, e := range epochs {
|
|
if e.Ended == nil && e.Epoch != st.Epoch {
|
|
open = append(open, fmt.Sprintf("epoch %d (%s)", e.Epoch, e.Instance))
|
|
}
|
|
}
|
|
if len(open) > 0 {
|
|
split("open-epochs", fmt.Sprintf("the mesh's record holds %s open beside this controller's epoch %d: a "+
|
|
"holder that neither gave the lease back nor was found expired", strings.Join(open, ", "), st.Epoch),
|
|
strings.Join(open, ", "))
|
|
}
|
|
// And no message from a stale epoch in the last interval.
|
|
for _, w := range link.StaleRefusals.Within(time.Now().Add(-doctorEvery)) {
|
|
if w.Epoch <= 0 || uint64(w.Epoch) >= st.Epoch {
|
|
continue
|
|
}
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: fmt.Sprintf("controller.epoch-%d", w.Epoch),
|
|
Token: "stale-epoch", Machine: firstOf(w.Receivers), Severity: conditions.Urgent,
|
|
Summary: fmt.Sprintf("%d declaration(s) from epoch %d — older than this controller's %d — reached %s in the "+
|
|
"last %s and were refused: a controller that lost the lease is still sending", w.Count, w.Epoch, st.Epoch,
|
|
strings.Join(w.Receivers, ", "), doctorEvery),
|
|
Said: fmt.Sprintf("%d refused, the last at %s", w.Count, w.Last.UTC().Format(time.RFC3339))})
|
|
}
|
|
return sortedFound(out), nil
|
|
}
|