// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6). // // **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every // one of the forty-eight core failures of research 031 was noticed because a person or an agent // looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is // raised when an observation says something is wrong past its bound, kept with since-when, evidence // and who can resolve it, said on the bus as it changes, and cleared when an observation says it is // resolved — never by hand. // // The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once — // the serving controller's watchdogs, and a command a person runs to silence one — so every write // is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes // itself rather than overwriting what the other said. package conditions import ( "fmt" "regexp" "sort" "strings" "time" ) // Severity is how soon the operator is needed: two levels, no more (to-be 45 §2). type Severity string const ( // Urgent needs the operator now. Urgent Severity = "urgent" // Warning needs the operator when they can. Warning Severity = "warning" ) // The scopes a condition's key starts with: what kind of thing is wrong. const ( ScopeMachine = "machine" ScopePlan = "plan" ScopeCall = "call" ScopeBuild = "build" ScopeMerge = "merge" ScopeProvider = "provider" ScopeSeat = "seat" ScopeBus = "bus" ScopeCore = "core" ScopeProbe = "probe" ScopeMesh = "mesh" ) // Scopes is every scope, in the order a person reads them. var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider, ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh} // Who resolves a condition. const ( // ResolverSelf clears on observation: the signal returns, the probe passes. ResolverSelf = "self" // ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy. ResolverOperator = "operator" // ResolverAgent is work handed to an agent (research 017; not raised by anything yet). ResolverAgent = "agent" ) // ResolverHealer is the resolver of a condition a registered healer works on (Phase 3). func ResolverHealer(name string) string { return "healer:" + name } // Subject is what the condition is about: its scope, its id within the scope, and the machine it // concerns when there is one. type Subject struct { Scope string `json:"scope"` ID string `json:"id"` Machine string `json:"machine,omitempty"` // Also are the other machines it concerns: a consumer's, for a provider failing it. Also []string `json:"also,omitempty"` } // Evidence is one observation, as it was said. type Evidence struct { At time.Time `json:"at"` Said string `json:"said"` } // Attempt is one healer's try at a condition (to-be 45 §7; written from Phase 3). type Attempt struct { At time.Time `json:"at"` What string `json:"what"` Outcome string `json:"outcome"` } // Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand // act; the condition stays open, and `status` still says it. type Silence struct { Until time.Time `json:"until"` By string `json:"by"` Why string `json:"why"` Since time.Time `json:"since"` } // KeptEvidence is how many observations a condition keeps, newest first. const KeptEvidence = 10 // MaxSilence is the longest a condition may be silenced at once: past it, a person says so again. const MaxSilence = 7 * 24 * time.Hour // ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its // count increased, rather than news (to-be 45 §2). const ReopenWithin = 10 * time.Minute // Condition is one open condition, as the store keeps it and its events carry it. type Condition struct { Key string `json:"key"` // Kind is the condition kind: from the signals table, the probe registry or an event kind. Kind string `json:"kind"` Subject Subject `json:"subject"` Severity Severity `json:"severity"` // Summary is one line in the mesh's words. Summary string `json:"summary"` // Evidence is the newest observations, at most KeptEvidence, newest first. Evidence []Evidence `json:"evidence"` // Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. Source string `json:"source"` // Raised is when it was first observed this time; LastObserved the newest observation. Raised time.Time `json:"raised"` LastObserved time.Time `json:"last-observed"` // Observations is how many times it was observed since raised. Observations int `json:"observations"` // Count is how many times it has been raised, a reopening within ReopenWithin counted. Count int `json:"count"` Tried []Attempt `json:"tried,omitempty"` Resolver string `json:"resolver"` // Silenced is null when no silence is in force: said, not left out, so a reader need not guess. Silenced *Silence `json:"silenced"` // Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written // by a controller serving without the lease, or before the lease existed. Epoch uint64 `json:"epoch"` } // SilencedAt says whether a person's silence is in force at a moment. func (c Condition) SilencedAt(now time.Time) bool { return c.Silenced != nil && now.Before(c.Silenced.Until) } // Show is the verb that shows more about a condition, as a message carries it. func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key } // Observation is one watchdog, probe or event saying something is wrong now. type Observation struct { Scope string // ID is the thing within the scope; several tokens joined by dots where the thing is named by // several (a provider's module, its machine and the consumer). ID string // Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a // provider. Kind's own word when empty. Token string Kind string Machine string // Also are the other machines it concerns. Also []string Severity Severity Summary string // Said is this observation's evidence, in the mesh's words; Summary when empty. Said string Source string Resolver string } // Key is where the observation's condition is kept: `..`, so the same fault said // again is the same condition. func (o Observation) Key() string { token := o.Token if token == "" { token = o.Kind } return Key(o.Scope, o.ID, token) } // unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's // tokens, and a `*` or `>` would make one a wildcard. var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) // Key composes a condition's key from its parts, each token made safe for the bus: a character the // bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about. func Key(scope, id, token string) string { var parts []string for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") if p == "" { p = "_" } parts = append(parts, p) } return strings.Join(parts, ".") } // check refuses an observation that could not be said: a condition with no kind, no scope the mesh // knows, or no severity is one nobody could route. func (o Observation) check() error { known := false for _, s := range Scopes { if s == o.Scope { known = true } } switch { case !known: return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope) case strings.TrimSpace(o.ID) == "": return fmt.Errorf("a %s condition names what it is about", o.Scope) case strings.TrimSpace(o.Kind) == "": return fmt.Errorf("the condition %s has no kind", o.Key()) case o.Severity != Urgent && o.Severity != Warning: return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity) case strings.TrimSpace(o.Summary) == "": return fmt.Errorf("the condition %s says nothing", o.Key()) case strings.TrimSpace(o.Source) == "": return fmt.Errorf("the condition %s does not say what raised it", o.Key()) } return nil } // Order sorts conditions as `status` says them: urgent before warning, then oldest first. func Order(list []Condition) { sort.SliceStable(list, func(i, j int) bool { if list[i].Severity != list[j].Severity { return list[i].Severity == Urgent } if !list[i].Raised.Equal(list[j].Raised) { return list[i].Raised.Before(list[j].Raised) } return list[i].Key < list[j].Key }) }