jochen
dcdafd37f0
Every look rewrote the condition's value in the conditions bucket, because the value carried the count of looks, the last look and the newest evidence: 131 647 writes in four days for twelve open conditions, against ADR 0289's rule that a bucket is written only when what it describes changed. The looks now live in the controller's store (migration 0095, condition_looks), the bucket holds what a condition says and is written only when that changes, and the conditions verb and the condition events put the looks back.
492 lines
16 KiB
Go
492 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// What is wrong, kept until observation says it is not (novox/hq to-be 45 §2, ADR 0227).
|
|
//
|
|
// **`status` used to be the only place the mesh said it was wrong, and only to whoever asked.** Every
|
|
// core failure of research 031 was found by a person looking. A condition is the mesh saying it: raised
|
|
// by a watchdog when a signal is late (signals.go), by a probe when an invariant does not hold
|
|
// (doctor.go), or by an event a provider sends (standing.go); kept on the bus with since-when, and its
|
|
// looks — how often, when last, the newest evidence — in the controller's store (condition_looks.go,
|
|
// novox/hq issue 420); said on the bus as it changes, for the operator's channel to carry; and cleared when an
|
|
// observation says it is resolved. Nobody resolves one by hand. A person who knows silences it, for a
|
|
// while, with a reason, and that is recorded as a hand act.
|
|
|
|
// conditionsFrom is the serving controller's keeper; nil in any other process, which opens its own.
|
|
var conditionsFrom *conditions.Keeper
|
|
|
|
// keeperOn is a keeper over the store on a connection, saying its transitions on that connection, with
|
|
// the conditions' looks kept in looks (condition_looks.go).
|
|
func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore) (*conditions.Keeper, error) {
|
|
store, history, err := conditions.OnTheBus(ctx, conn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
js, err := conn.JetStream()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return conditions.NewKeeper(ctx, conditions.Options{Store: store, History: history, Looks: looks,
|
|
Teller: link.OverNATS{Conn: conn, JS: js},
|
|
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
|
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
|
// does nothing).
|
|
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
|
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
|
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
|
}
|
|
|
|
// withKeeper runs f with the serving controller's keeper, or one of its own that says everything
|
|
// it was given before it returns.
|
|
func withKeeper(ctx context.Context, f func(*conditions.Keeper) error) error {
|
|
if conditionsFrom != nil {
|
|
return f(conditionsFrom)
|
|
}
|
|
looks, closeLooks := looksHere(ctx)
|
|
defer closeLooks()
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
k, err := keeperOn(ctx, conn, looks)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
flushing, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
k.Close(flushing)
|
|
}()
|
|
return f(k)
|
|
})
|
|
}
|
|
|
|
// openConditions is every open condition, for `status` and `node show`: from the serving keeper, or
|
|
// read from the bus. Where there is no bus to read it from, it says so — never "none open".
|
|
func openConditions(ctx context.Context) ([]conditions.Condition, error) {
|
|
if conditionsFrom != nil {
|
|
return conditionsFrom.Open(ctx)
|
|
}
|
|
if _, err := broker.BusAddress(); err != nil {
|
|
return nil, fmt.Errorf("this process has no bus to read the conditions from: %w", err)
|
|
}
|
|
var out []conditions.Condition
|
|
looks, closeLooks := looksHere(ctx)
|
|
defer closeLooks()
|
|
err := onTheBus(func(conn *nats.Conn) error {
|
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
out, err = conditions.ReadWith(reading, store, looks)
|
|
return err
|
|
})
|
|
return out, err
|
|
}
|
|
|
|
// conditionsUsage is how the verb is typed.
|
|
const conditionsUsage = "conditions [--scope S] [--severity urgent|warning] [--machine M] [--json] | " +
|
|
"conditions show <key> | conditions silence <key> --for <duration> --why <text> | " +
|
|
"conditions history [--days N] [--key K] [--json]"
|
|
|
|
// conditionsCommand is `conditions`, `conditions show`, `conditions silence` and `conditions history`.
|
|
func conditionsCommand(ctx context.Context, args []string) error {
|
|
sub := "list"
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
sub, args = args[0], args[1:]
|
|
}
|
|
switch sub {
|
|
case "list":
|
|
return listConditions(ctx, args, os.Stdout)
|
|
case "show":
|
|
return showCondition(ctx, args, os.Stdout)
|
|
case "silence":
|
|
return silenceCondition(ctx, args)
|
|
case "history":
|
|
return conditionHistory(ctx, args, os.Stdout)
|
|
}
|
|
return errors.New(conditionsUsage)
|
|
}
|
|
|
|
func listConditions(ctx context.Context, args []string, w io.Writer) error {
|
|
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
|
|
usageTo(set, w)
|
|
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
|
|
severity := set.String("severity", "", "only urgent, or only warning")
|
|
machine := set.String("machine", "", "only those about this machine")
|
|
asJSON := set.Bool("json", false, "as data")
|
|
if rest, err := parseAround(set, args); err != nil {
|
|
return err
|
|
} else if len(rest) > 0 {
|
|
return errors.New(conditionsUsage)
|
|
}
|
|
if *severity != "" && *severity != string(conditions.Urgent) && *severity != string(conditions.Warning) {
|
|
return fmt.Errorf("a severity is urgent or warning, not %q", *severity)
|
|
}
|
|
open, err := openConditions(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var out []conditions.Condition
|
|
for _, c := range open {
|
|
if (*scope == "" || c.Subject.Scope == *scope) && (*severity == "" || string(c.Severity) == *severity) &&
|
|
(*machine == "" || concerns(c, *machine)) {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
if *asJSON {
|
|
return printJSONTo(w, map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
|
|
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; " +
|
|
"each with its newest evidence — `conditions key=<key>` gives one whole"})
|
|
}
|
|
if len(out) == 0 {
|
|
if len(open) == 0 {
|
|
fmt.Fprintln(w, "no open conditions")
|
|
} else {
|
|
fmt.Fprintf(w, "none of the %d open condition(s) is about that\n", len(open))
|
|
}
|
|
return nil
|
|
}
|
|
for _, line := range conditionLines(out, time.Now()) {
|
|
fmt.Fprintln(w, line)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// briefEvidence is how many observations a condition carries in a list of them: its newest. The store
|
|
// keeps ten, and a list of hundreds of conditions with ten each was the larger half of an answer more
|
|
// than the bus carries in one message (novox/hq issue 314). One condition whole is `conditions key=`.
|
|
const briefEvidence = 1
|
|
|
|
// conditionInBrief is a condition as a list carries it: its newest evidence, with how much more there is
|
|
// and where to read it. Everything else of it, so a reader of the list — the operator's channel among
|
|
// them — reads the same condition it always did.
|
|
type conditionInBrief struct {
|
|
conditions.Condition
|
|
// EvidenceKept is how many observations the condition keeps, when the list shows fewer.
|
|
EvidenceKept int `json:"evidence-kept,omitempty"`
|
|
More string `json:"more,omitempty"`
|
|
}
|
|
|
|
// inBrief is a list of conditions in brief; never null.
|
|
func inBrief(list []conditions.Condition) []conditionInBrief {
|
|
out := make([]conditionInBrief, 0, len(list))
|
|
for _, c := range list {
|
|
b := conditionInBrief{Condition: c}
|
|
if len(c.Evidence) > briefEvidence {
|
|
b.EvidenceKept = len(c.Evidence)
|
|
b.Evidence = c.Evidence[:briefEvidence:briefEvidence]
|
|
b.More = c.Show()
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// counted is how many of a list each source raised of each kind — `D14 stalled: 689` — so a list of
|
|
// hundreds says its shape before its lines.
|
|
func counted(list []conditions.Condition) map[string]int {
|
|
out := map[string]int{}
|
|
for _, c := range list {
|
|
out[c.Source+" "+c.Kind]++
|
|
}
|
|
return out
|
|
}
|
|
|
|
// concerns says whether a condition is about a machine: it names it, or its key does.
|
|
func concerns(c conditions.Condition, machine string) bool {
|
|
if c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) {
|
|
return true
|
|
}
|
|
for _, part := range strings.Split(c.Key, ".") {
|
|
if part == machine {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// conditionLines is how a list of conditions reads: one line each, its silence under it.
|
|
func conditionLines(list []conditions.Condition, now time.Time) []string {
|
|
var out []string
|
|
for _, c := range list {
|
|
times := ""
|
|
if c.Count > 1 {
|
|
times = fmt.Sprintf(", raised %d times", c.Count)
|
|
}
|
|
out = append(out, fmt.Sprintf(" %-7s %s — %s (since %s%s)", strings.ToUpper(string(c.Severity)),
|
|
c.Key, c.Summary, c.Raised.Local().Format("2006-01-02 15:04"), times))
|
|
if c.SilencedAt(now) {
|
|
out = append(out, fmt.Sprintf(" silenced until %s by %s: %s",
|
|
c.Silenced.Until.Local().Format("2006-01-02 15:04"), c.Silenced.By, c.Silenced.Why))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func showCondition(ctx context.Context, args []string, w io.Writer) error {
|
|
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
|
|
usageTo(set, w)
|
|
asJSON := set.Bool("json", false, "as data")
|
|
rest, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(rest) != 1 {
|
|
return errors.New("conditions show <key>")
|
|
}
|
|
key := rest[0]
|
|
var c conditions.Condition
|
|
var found bool
|
|
if conditionsFrom != nil {
|
|
c, found, err = conditionsFrom.Get(ctx, key)
|
|
} else {
|
|
looks, closeLooks := looksHere(ctx)
|
|
defer closeLooks()
|
|
err = onTheBus(func(conn *nats.Conn) error {
|
|
store, _, err := conditions.OnTheBus(ctx, conn)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c, found, err = conditions.ReadOneWith(ctx, store, looks, key)
|
|
return err
|
|
})
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("no condition %s is open — `conditions` lists those that are, and `conditions "+
|
|
"history --key %s` what became of it", key, key)
|
|
}
|
|
if *asJSON {
|
|
return printJSONTo(w, c)
|
|
}
|
|
now := time.Now()
|
|
fmt.Fprintf(w, "%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
|
|
fmt.Fprintf(w, " kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
|
|
if c.Subject.Machine != "" {
|
|
fmt.Fprintf(w, ", on %s", c.Subject.Machine)
|
|
}
|
|
fmt.Fprintf(w, "\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
|
|
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
|
|
now.Sub(c.LastObserved).Round(time.Second))
|
|
if c.Count > 1 {
|
|
fmt.Fprintf(w, " raised %d times, each within ten minutes of clearing\n", c.Count)
|
|
}
|
|
fmt.Fprintf(w, " resolved by %s\n", resolverWords(c.Resolver))
|
|
if c.Silenced != nil {
|
|
fmt.Fprintf(w, " silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
|
|
c.Silenced.By, c.Silenced.Why)
|
|
}
|
|
if len(c.Tried) > 0 {
|
|
fmt.Fprintln(w, "\n tried:")
|
|
for _, t := range c.Tried {
|
|
fmt.Fprintf(w, " %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
|
|
}
|
|
}
|
|
fmt.Fprintln(w, "\n evidence, newest first:")
|
|
for _, e := range c.Evidence {
|
|
fmt.Fprintf(w, " %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func resolverWords(r string) string {
|
|
switch r {
|
|
case conditions.ResolverSelf:
|
|
return "itself: it clears when observation says it is resolved"
|
|
case conditions.ResolverOperator:
|
|
return "the operator: nothing in the mesh will repair it"
|
|
case conditions.ResolverAgent:
|
|
return "an agent"
|
|
}
|
|
return r
|
|
}
|
|
|
|
// silenceCondition stops a condition's messages for a while (to-be 45 §2). A hand act: recorded with
|
|
// who and why before it is done, its cause the condition's kind unless one is given.
|
|
func silenceCondition(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("conditions silence", flag.ContinueOnError)
|
|
forFlag := set.String("for", "", "how long: 30m, 4h, 2d — at most 7d")
|
|
acts := addHandActFlags(set)
|
|
rest, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(rest) != 1 {
|
|
return errors.New("conditions silence <key> --for <duration> --why <text>")
|
|
}
|
|
key := rest[0]
|
|
if err := acts.require("conditions silence"); err != nil {
|
|
return err
|
|
}
|
|
d, err := parseFor(*forFlag)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d > conditions.MaxSilence {
|
|
return fmt.Errorf("a condition is silenced for at most %s at once; past it, say so again", conditions.MaxSilence)
|
|
}
|
|
return withKeeper(ctx, func(k *conditions.Keeper) error {
|
|
c, found, err := k.Get(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("no condition %s is open — `conditions` lists them. Nothing was silenced", key)
|
|
}
|
|
if strings.TrimSpace(*acts.cause) == "" {
|
|
*acts.cause = c.Kind
|
|
}
|
|
*acts.condition = key
|
|
acts.record(ctx, "conditions silence", []string{key, "--for", *forFlag})
|
|
held, err := k.Silence(ctx, key, d, link.Caller(), *acts.why)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s is silenced until %s: no message is sent for it until then. It is still open, and "+
|
|
"`status` still says it; it clears when observation says it is resolved\n",
|
|
held.Key, held.Silenced.Until.Local().Format("2006-01-02 15:04"))
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// parseFor reads a duration, days included.
|
|
func parseFor(s string) (time.Duration, error) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0, errors.New("say for how long: --for 30m, 4h or 2d")
|
|
}
|
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
|
n, err := strconv.Atoi(days)
|
|
if err != nil || n <= 0 {
|
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
|
}
|
|
return time.Duration(n) * 24 * time.Hour, nil
|
|
}
|
|
d, err := time.ParseDuration(s)
|
|
if err != nil || d <= 0 {
|
|
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
func conditionHistory(ctx context.Context, args []string, w io.Writer) error {
|
|
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
|
|
usageTo(set, w)
|
|
days := set.Int("days", 7, "how many days back, at most 90")
|
|
key := set.String("key", "", "only this condition")
|
|
asJSON := set.Bool("json", false, "as data")
|
|
if rest, err := parseAround(set, args); err != nil {
|
|
return err
|
|
} else if len(rest) > 0 {
|
|
return errors.New("conditions history [--days N] [--key K] [--json]")
|
|
}
|
|
since := time.Now().Add(-time.Duration(*days) * 24 * time.Hour)
|
|
var events []conditions.Event
|
|
read := func(h conditions.History) error {
|
|
var err error
|
|
events, err = h.Since(ctx, since)
|
|
return err
|
|
}
|
|
var err error
|
|
if conditionsFrom != nil {
|
|
events, err = conditionsFrom.HistorySince(ctx, since)
|
|
} else {
|
|
err = onTheBus(func(conn *nats.Conn) error {
|
|
_, history, err := conditions.OnTheBus(ctx, conn)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return read(history)
|
|
})
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var out []conditions.Event
|
|
for _, e := range events {
|
|
if *key == "" || e.Key == *key {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
if *asJSON {
|
|
if out == nil {
|
|
out = []conditions.Event{}
|
|
}
|
|
return printJSONTo(w, map[string]any{"history": out, "days": *days})
|
|
}
|
|
if len(out) == 0 {
|
|
fmt.Fprintf(w, "nothing was raised, changed or cleared in the last %d day(s)\n", *days)
|
|
return nil
|
|
}
|
|
for _, e := range out {
|
|
line := fmt.Sprintf("%s %-13s %s", e.At.Local().Format("2006-01-02 15:04:05"), e.Change, e.Key)
|
|
switch e.Change {
|
|
case conditions.ChangeRaised, conditions.ChangeReopened:
|
|
line += " — " + e.Summary
|
|
case conditions.ChangeSeverity:
|
|
line += fmt.Sprintf(" — %s, was %s", e.Severity, e.Was)
|
|
case conditions.ChangeResolver:
|
|
line += fmt.Sprintf(" — %s, was %s", e.Resolver, e.Was)
|
|
default:
|
|
if e.Why != "" {
|
|
line += " — " + e.Why
|
|
}
|
|
}
|
|
fmt.Fprintln(w, line)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// printConditions is the status section that leads it: every open condition, urgent first, oldest
|
|
// first, silenced ones with their expiry (to-be 45 §2). A store that could not be read is said, and
|
|
// is not "none open".
|
|
func printConditions(list []conditions.Condition, unread string, now time.Time) {
|
|
if unread != "" {
|
|
fmt.Printf("the open conditions could NOT be read, so whether anything is wrong is not known: %s\n\n", unread)
|
|
return
|
|
}
|
|
if len(list) == 0 {
|
|
return
|
|
}
|
|
urgent := 0
|
|
for _, c := range list {
|
|
if c.Severity == conditions.Urgent {
|
|
urgent++
|
|
}
|
|
}
|
|
fmt.Printf("%d open condition(s), %d urgent:\n\n", len(list), urgent)
|
|
for _, line := range conditionLines(list, now) {
|
|
fmt.Println(line)
|
|
}
|
|
fmt.Printf("\n `conditions show <key>` says more; each clears when observation says it is resolved, " +
|
|
"never by hand — `conditions silence <key> --for <d> --why <text>` stops its messages\n\n")
|
|
}
|
|
|
|
// orHealer is who tried, as an attempt names it.
|
|
func orHealer(by string) string {
|
|
if by == "" {
|
|
return "a healer"
|
|
}
|
|
return by
|
|
}
|