package keeper import ( "fmt" "strings" "time" ) // Bound is how long the keeper may go without its fill pass, or leave an accepted verdict unanswered, before it is // unhealthy (design 25 ยง 12.2): three minutes, so the only guard against a store filling never stops without a sound. const Bound = 3 * time.Minute // Health is the keeper's health as its tool answers it: healthy or not, why, and what it names beside (the entries it // could not create, the ones waiting, the ones past the raise mark). Another module's entry past the raise mark raises // that module's condition and leaves the keeper healthy. type Health struct { Healthy bool `json:"healthy"` Why string `json:"why"` Pass string `json:"pass"` Refused []string `json:"refused,omitempty"` Waiting []string `json:"waiting,omitempty"` Full []string `json:"full,omitempty"` } // HealthOf judges the keeper from its status, what it holds, and pending: how many accepted verdicts wait on its // durable consumer unanswered (-1 when it could not be read). Pure. func HealthOf(s Status, held map[string]Asked, pending int, now time.Time) Health { h := Health{Healthy: true} var why []string switch { case s.Pass.IsZero() && (s.Started.IsZero() || now.Sub(s.Started) > Bound): why = append(why, "the keeper has made no fill pass since its start") h.Pass = "none" case !s.Pass.IsZero(): h.Pass = s.Pass.Format(time.RFC3339) if now.Sub(s.Pass) > Bound { w := fmt.Sprintf("its last fill pass is %s old, more than %s", now.Sub(s.Pass).Round(time.Second), Bound) if len(s.Unsaid)+len(s.UnsaidConditions) > 0 { // Its status may be what is not written: what it held unsaid then may be lost at a restart. w += fmt.Sprintf(", so its status may no longer be written: the %d word(s) it held unsaid then, and any since, are kept only in its memory and may be lost at a restart", len(s.Unsaid)+len(s.UnsaidConditions)) } why = append(why, w) } } // Its first view unanswered past the bound since its start, not at once: the indexer answers within it. if s.FirstView.Answered.IsZero() && (s.Started.IsZero() || now.Sub(s.Started) > Bound) { w := fmt.Sprintf("its first view, %s.%s, has not answered", s.FirstView.Owner, s.FirstView.Verb) if s.FirstView.Error != "" { w += ": " + s.FirstView.Error } why = append(why, w) } switch { case pending < 0: // Never skipped silently: what waits on its consumer is not known, so nothing says it is answered. why = append(why, "what waits on its consumer cannot be read from the bus's monitoring") case pending > 0 && (s.Heard.IsZero() || now.Sub(s.Heard) > Bound): why = append(why, fmt.Sprintf("%d accepted verdict(s) wait on its consumer unanswered for more than %s", pending, Bound)) } if n := len(s.Unsaid) + len(s.UnsaidConditions); n > 0 { var what []string for _, u := range s.Unsaid { what = append(what, fmt.Sprintf("%s of %s %s (%s)", u.Event, u.Body.Module, u.Body.Local, u.Error)) } for _, key := range sortedKeys(s.UnsaidConditions) { what = append(what, "the condition "+key+" ("+s.UnsaidConditions[key].State+")") } why = append(why, fmt.Sprintf("%d word(s) the bus did not take wait to be said again at the next fill pass: %s", n, strings.Join(what, "; "))) } for _, module := range sortedKeys(held) { for _, local := range sortedKeys(held[module].Made) { m := held[module].Made[local] name := BusName(module, local) switch { case m.Refused != "": h.Refused = append(h.Refused, name+": "+m.Refused) case m.Waiting != "": h.Waiting = append(h.Waiting, name+": "+m.Waiting) } if m.Full { h.Full = append(h.Full, fmt.Sprintf("%s: %d%%", name, m.Fill)) } } } if len(why) > 0 { h.Healthy, h.Why = false, strings.Join(why, "; ") } else { h.Why = "its fill pass runs and its first view is held" } return h }