mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…
One commit with exactly the tree of the reviewed head35b4509c0a, on main12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).
382 lines
13 KiB
Go
382 lines
13 KiB
Go
package main
|
|
|
|
// The controller's condition events, read in this one place (novox/hq to-be 45 §2).
|
|
//
|
|
// **The contract this reads, and every assumption it makes about it**, because the controller's
|
|
// side was built at the same time from the same design and the design names the events and the
|
|
// fields but not their spelling on the wire:
|
|
//
|
|
// - The events are the mesh-controller seat's own: `condition-raised`, `condition-changed` and
|
|
// `condition-cleared`, consumed as `mesh-controller.<event>` (subject
|
|
// `mesh.seat.mesh-controller.event.<event>`), exactly as `applied` and `built-before` are.
|
|
// - The body is the condition as the store holds it, one JSON object, with the field names of the
|
|
// design's table in kebab-case: key, kind, subject, severity, summary, evidence, source, raised,
|
|
// last-observed, observations, tried, resolver, silenced, epoch. snake_case and camelCase
|
|
// spellings of the two-word names are read too.
|
|
// - `subject` is an object {scope, id, machine}; a plain string is read as well.
|
|
// - `severity` is `urgent` or `warning`. Anything else is not guessed: the event is unreadable.
|
|
// - `silenced` is absent, null, or an object {until, by, why}; `until` in the past is not silenced.
|
|
// - Times are RFC 3339.
|
|
// - `headline`, `explanation` and `resolved` are the plain words the operator reads (novox/hq ADR
|
|
// 0253): a condition from a controller older than them has none, and is said from its summary.
|
|
// - `needs` is what the operator does, empty for nothing (the explanation opens with the verdict);
|
|
// `actions` are the answers a notification offers, each {label, verb, machine, arguments}.
|
|
// - `key` is `<scope>.<id>.<kind>`. When it is absent it is made from subject and kind; when
|
|
// neither gives one the event is unreadable.
|
|
// - A cleared event carries the condition as last held, and may add `cleared` (its time).
|
|
// - Every event carries `at`, when the transition happened (the controller's conditions.Event).
|
|
// It is how this holder tells news from history: a stream replays what it holds to a consumer
|
|
// that is new or was away, and an event's delivery time says nothing about when it happened.
|
|
// - The `conditions` verb of the controller's seat answers `{conditions: [...], open, note}`, each
|
|
// entry the condition as the store holds it — the same shape as an event's body.
|
|
//
|
|
// An event this cannot read is refused by name — which event, which field, why — counted, said in
|
|
// the status and in the log, and told to the operator; never read as an empty condition.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
)
|
|
|
|
// The events, by their local names under the controller's seat.
|
|
const (
|
|
EventRaised = "condition-raised"
|
|
EventChanged = "condition-changed"
|
|
EventCleared = "condition-cleared"
|
|
// ControllerSeat is the role the events are stated under.
|
|
ControllerSeat = "mesh-controller"
|
|
)
|
|
|
|
// Severities: two levels, no more (to-be 45 §2).
|
|
const (
|
|
Urgent = "urgent"
|
|
Warning = "warning"
|
|
)
|
|
|
|
// Condition is what this holder needs of one.
|
|
type Condition struct {
|
|
Key string
|
|
Kind string
|
|
Scope string
|
|
ID string
|
|
Machine string
|
|
Severity string
|
|
Summary string
|
|
Headline string
|
|
Explanation string
|
|
Resolved string
|
|
Needs string
|
|
Actions []Action
|
|
Source string
|
|
// Publisher is the module that said the condition on the operator channel's one subject; empty for the
|
|
// controller's own, which a reading of the controller's conditions may end.
|
|
Publisher string
|
|
Resolver string
|
|
Raised time.Time
|
|
LastObserved time.Time
|
|
Observations int
|
|
SilencedTill time.Time
|
|
SilencedWhy string
|
|
Cleared time.Time
|
|
// At is when the transition the event says happened; zero for a condition read from the open set.
|
|
At time.Time
|
|
}
|
|
|
|
// When is the event's own time: `at`; without it, the clearing's or the raising's time; without
|
|
// those, zero — and a zero time is read as now by the holder, never as old.
|
|
func (c Condition) When(event string) time.Time {
|
|
switch {
|
|
case !c.At.IsZero():
|
|
return c.At
|
|
case event == EventCleared:
|
|
return c.Cleared
|
|
case event == EventRaised:
|
|
return c.Raised
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
// SubjectWords is the subject as a person reads it: "machine ace", "plan 41", "provider keycloak".
|
|
func (c Condition) SubjectWords() string {
|
|
parts := []string{}
|
|
if c.Scope != "" {
|
|
parts = append(parts, c.Scope)
|
|
}
|
|
if c.ID != "" {
|
|
parts = append(parts, c.ID)
|
|
}
|
|
if c.Machine != "" && c.Machine != c.ID {
|
|
parts = append(parts, "on "+c.Machine)
|
|
}
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
// SilencedAt says whether the condition's messages are stopped at that moment.
|
|
func (c Condition) SilencedAt(now time.Time) bool {
|
|
return !c.SilencedTill.IsZero() && now.Before(c.SilencedTill)
|
|
}
|
|
|
|
// eventOf is the local event name of an envelope's key: `mesh-controller.condition-raised` is
|
|
// `condition-raised`. Anything not of the controller's seat is not a condition event.
|
|
func eventOf(key string) (string, bool) { return eventOfSeat(key, ControllerSeat) }
|
|
|
|
// eventOfSeat is eventOf for the seat the take-over key says is followed: the controller's, or the keeper's (ADR 0564 clause 9).
|
|
func eventOfSeat(key, seat string) (string, bool) {
|
|
emitter, event, ok := strings.Cut(key, ".")
|
|
if !ok || emitter != seat {
|
|
return "", false
|
|
}
|
|
switch event {
|
|
case EventRaised, EventChanged, EventCleared:
|
|
return event, true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// DecodeCondition reads one condition event's body.
|
|
func DecodeCondition(event string, body []byte) (Condition, error) {
|
|
var raw map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &raw); err != nil {
|
|
return Condition{}, fmt.Errorf("%s: the body is not a JSON object: %v", event, err)
|
|
}
|
|
if raw == nil {
|
|
return Condition{}, fmt.Errorf("%s: the body is null", event)
|
|
}
|
|
var c Condition
|
|
var err error
|
|
str := func(names ...string) string {
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
for _, n := range names {
|
|
v, ok := raw[n]
|
|
if !ok || string(v) == "null" {
|
|
continue
|
|
}
|
|
var s string
|
|
if e := json.Unmarshal(v, &s); e != nil {
|
|
err = fmt.Errorf("%s: %s is not a string", event, n)
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(s)
|
|
}
|
|
return ""
|
|
}
|
|
when := func(names ...string) time.Time {
|
|
s := str(names...)
|
|
if s == "" || err != nil {
|
|
return time.Time{}
|
|
}
|
|
t, e := time.Parse(time.RFC3339Nano, s)
|
|
if e != nil {
|
|
err = fmt.Errorf("%s: %s is not an RFC 3339 time: %q", event, names[0], s)
|
|
}
|
|
return t
|
|
}
|
|
c.Key = str("key")
|
|
c.Kind = str("kind")
|
|
c.Severity = str("severity")
|
|
c.Summary = str("summary")
|
|
c.Headline = str("headline")
|
|
c.Explanation = str("explanation")
|
|
c.Resolved = str("resolved")
|
|
c.Needs = str("needs")
|
|
if v, ok := raw["actions"]; ok && string(v) != "null" {
|
|
if e := json.Unmarshal(v, &c.Actions); e != nil {
|
|
return Condition{}, fmt.Errorf("%s: actions is not a list of {label, verb, machine, arguments}", event)
|
|
}
|
|
}
|
|
c.Resolver = str("resolver")
|
|
c.Raised = when("raised")
|
|
c.LastObserved = when("last-observed", "last_observed", "lastObserved")
|
|
c.Cleared = when("cleared")
|
|
c.At = when("at")
|
|
if err != nil {
|
|
return Condition{}, err
|
|
}
|
|
// source: a string, or an object naming the row, probe or event.
|
|
if v, ok := raw["source"]; ok && string(v) != "null" {
|
|
var s string
|
|
if json.Unmarshal(v, &s) == nil {
|
|
c.Source = s
|
|
} else {
|
|
var o map[string]any
|
|
if json.Unmarshal(v, &o) == nil {
|
|
for _, k := range []string{"row", "probe", "event", "name", "id"} {
|
|
if s, ok := o[k].(string); ok && s != "" {
|
|
c.Source = s
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, n := range []string{"observations", "count"} {
|
|
if v, ok := raw[n]; ok && string(v) != "null" {
|
|
var f float64
|
|
if json.Unmarshal(v, &f) != nil {
|
|
return Condition{}, fmt.Errorf("%s: %s is not a number", event, n)
|
|
}
|
|
c.Observations = int(f)
|
|
break
|
|
}
|
|
}
|
|
if v, ok := raw["subject"]; ok && string(v) != "null" {
|
|
var s string
|
|
if json.Unmarshal(v, &s) == nil {
|
|
c.ID = strings.TrimSpace(s)
|
|
} else {
|
|
var o struct {
|
|
Scope string `json:"scope"`
|
|
ID string `json:"id"`
|
|
Machine string `json:"machine"`
|
|
Node string `json:"node"`
|
|
}
|
|
if e := json.Unmarshal(v, &o); e != nil {
|
|
return Condition{}, fmt.Errorf("%s: subject is neither a string nor {scope, id, machine}", event)
|
|
}
|
|
c.Scope, c.ID, c.Machine = o.Scope, o.ID, o.Machine
|
|
if c.Machine == "" {
|
|
c.Machine = o.Node
|
|
}
|
|
}
|
|
}
|
|
if v, ok := raw["silenced"]; ok && string(v) != "null" && string(v) != "{}" && string(v) != `""` && string(v) != "false" {
|
|
var o struct {
|
|
Until string `json:"until"`
|
|
Why string `json:"why"`
|
|
}
|
|
if e := json.Unmarshal(v, &o); e != nil {
|
|
return Condition{}, fmt.Errorf("%s: silenced is not {until, by, why}", event)
|
|
}
|
|
if o.Until != "" {
|
|
t, e := time.Parse(time.RFC3339Nano, o.Until)
|
|
if e != nil {
|
|
return Condition{}, fmt.Errorf("%s: silenced.until is not an RFC 3339 time: %q", event, o.Until)
|
|
}
|
|
c.SilencedTill = t
|
|
}
|
|
c.SilencedWhy = o.Why
|
|
}
|
|
if c.Key == "" && c.Scope != "" && c.ID != "" && c.Kind != "" {
|
|
c.Key = c.Scope + "." + c.ID + "." + c.Kind
|
|
}
|
|
if c.Key == "" {
|
|
return Condition{}, fmt.Errorf("%s: no key, and no subject and kind to make one from", event)
|
|
}
|
|
if c.Scope == "" || c.ID == "" || c.Kind == "" {
|
|
// The key names them (`<scope>.<id>.<kind>`, the id itself possibly dotted).
|
|
parts := strings.Split(c.Key, ".")
|
|
if len(parts) >= 3 {
|
|
if c.Scope == "" {
|
|
c.Scope = parts[0]
|
|
}
|
|
if c.Kind == "" {
|
|
c.Kind = parts[len(parts)-1]
|
|
}
|
|
if c.ID == "" {
|
|
c.ID = strings.Join(parts[1:len(parts)-1], ".")
|
|
}
|
|
}
|
|
}
|
|
switch c.Severity {
|
|
case Urgent, Warning:
|
|
case "":
|
|
if event != EventCleared {
|
|
return Condition{}, fmt.Errorf("%s %s: no severity", event, c.Key)
|
|
}
|
|
default:
|
|
return Condition{}, fmt.Errorf("%s %s: severity %q is neither urgent nor warning", event, c.Key, c.Severity)
|
|
}
|
|
if c.Summary == "" && event != EventCleared {
|
|
return Condition{}, fmt.Errorf("%s %s: no summary", event, c.Key)
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// DecodeOpen reads the controller's `conditions` answer — what is open now — bare, inside the
|
|
// runtime's `{answer}` or `{output}`, inside a tool reply's text content, or as a JSON string. An
|
|
// answer without a `conditions` list is refused: an empty open set is said, never assumed.
|
|
func DecodeOpen(raw json.RawMessage) ([]Condition, error) {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &m); err != nil {
|
|
var s string
|
|
if json.Unmarshal(raw, &s) == nil {
|
|
return DecodeOpen(json.RawMessage(s))
|
|
}
|
|
return nil, fmt.Errorf("the conditions answer is not JSON")
|
|
}
|
|
if list, ok := m["conditions"]; ok {
|
|
var items []json.RawMessage
|
|
if string(list) != "null" {
|
|
if err := json.Unmarshal(list, &items); err != nil {
|
|
return nil, fmt.Errorf("the conditions answer's conditions is not a list")
|
|
}
|
|
}
|
|
out := make([]Condition, 0, len(items))
|
|
for _, it := range items {
|
|
c, err := DecodeCondition("conditions", it)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, c)
|
|
}
|
|
return out, nil
|
|
}
|
|
for _, k := range []string{"answer", "output"} {
|
|
if v, ok := m[k]; ok && string(v) != "null" {
|
|
return DecodeOpen(v)
|
|
}
|
|
}
|
|
if v, ok := m["content"]; ok {
|
|
var content []struct {
|
|
Text string `json:"text"`
|
|
}
|
|
var e struct {
|
|
IsError bool `json:"isError"`
|
|
}
|
|
_ = json.Unmarshal(raw, &e)
|
|
if json.Unmarshal(v, &content) == nil && len(content) > 0 {
|
|
if e.IsError {
|
|
return nil, fmt.Errorf("the controller refused: %s", content[0].Text)
|
|
}
|
|
return DecodeOpen(json.RawMessage(content[0].Text))
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("the conditions answer lists no conditions")
|
|
}
|
|
|
|
// ConditionVerb is the verb a module says a condition with: `mesh.seat.operator-channel.accept.condition.<publisher>`
|
|
// (novox/hq ADR 0437 §9).
|
|
const ConditionVerb = "condition"
|
|
|
|
// takeCondition takes one condition a module said on the seat's one subject. It is decided exactly as the
|
|
// controller's conditions are, by Condition: there is no second path. The publisher is the subject's last
|
|
// token, which the bus grants that module alone, so it is not compared with the condition's key. A body that is
|
|
// not a condition in the SDK's layout is logged by publisher, taken and dropped.
|
|
func (h *Holder) takeCondition(publisher string, body []byte) {
|
|
sc, err := asks.DecodeCondition(body)
|
|
if err != nil {
|
|
h.Logf("[messenger] a condition from %s was taken and dropped: %v", publisher, err)
|
|
return
|
|
}
|
|
c := Condition{Key: sc.Key, Scope: sc.Subject.Scope, ID: sc.Subject.ID, Machine: sc.Subject.Machine,
|
|
Severity: sc.Severity, Summary: sc.Summary, Headline: sc.Headline, Explanation: sc.Explanation,
|
|
Resolved: sc.Resolved, Needs: sc.Needs, Source: publisher, Publisher: publisher}
|
|
if i := strings.LastIndex(sc.Key, "."); i >= 0 {
|
|
c.Kind = sc.Key[i+1:]
|
|
}
|
|
for _, a := range sc.Actions {
|
|
c.Actions = append(c.Actions, Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments})
|
|
}
|
|
if sc.State == asks.StateCleared {
|
|
h.Condition(EventCleared, c)
|
|
return
|
|
}
|
|
h.Condition(EventRaised, c)
|
|
}
|