165 lines
6.7 KiB
Go
165 lines
6.7 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
|
|
// for longer than one apply interval (novox/hq issue 233).
|
|
const kindApplyFailing = "apply-failing"
|
|
|
|
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
|
|
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
|
|
const applyEvery = 5 * time.Minute
|
|
|
|
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
|
|
// (mesh-host's probe.go): "the package manager (pacman)".
|
|
const packageManagerKey = "the package manager ("
|
|
|
|
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
|
|
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
|
|
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
|
|
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
|
|
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
|
|
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
|
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
|
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
|
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
|
heard := map[string]bool{}
|
|
for _, m := range f.machines {
|
|
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
|
heard[m.name] = true
|
|
}
|
|
}
|
|
var out []conditions.Observation
|
|
said := map[string]bool{}
|
|
for _, d := range f.doing {
|
|
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
|
continue
|
|
}
|
|
out = append(out, applyFailing(d, f.now))
|
|
said[d.Node] = true
|
|
}
|
|
// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
|
|
// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
|
|
// on a machine whose node-engine applied what it was sent.
|
|
failed := map[string][]inventory.ProvReport{}
|
|
for _, r := range f.provFailed {
|
|
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
|
|
continue
|
|
}
|
|
failed[r.Node] = append(failed[r.Node], r)
|
|
}
|
|
for _, node := range sortedNames(failed) {
|
|
out = append(out, provisioningFailing(node, failed[node], f.now))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
|
|
// place.
|
|
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
|
|
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
|
|
var names, words []string
|
|
since := failed[0].At
|
|
for _, r := range failed {
|
|
names = append(names, r.Module)
|
|
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
|
|
if r.At.Before(since) {
|
|
since = r.At
|
|
}
|
|
}
|
|
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
|
Severity: conditions.Warning, Said: strings.Join(words, "; "),
|
|
Resolved: m + " applies what it is sent again"}
|
|
o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
|
|
strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
|
|
o.Headline = m + " is not applying its update"
|
|
o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
|
|
"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
|
|
return o
|
|
}
|
|
|
|
func orNoReason(s string) string {
|
|
if s == "" {
|
|
return "no reason given"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// applyFailing is one machine's S21 condition.
|
|
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
|
m := d.Node
|
|
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
|
|
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
|
Severity: conditions.Warning, Said: doingWords(d),
|
|
Resolved: m + " applies what it is sent again"}
|
|
how := "refused its declaration whole"
|
|
if d.Outcome != inventory.OutcomeRefused {
|
|
how = "applied its declaration only in part"
|
|
}
|
|
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
|
|
o.Headline = m + " is not applying its update"
|
|
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
|
|
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
|
|
if tool := unansweringTool(d); tool != "" {
|
|
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
|
|
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
|
|
o.Headline = fmt.Sprintf("%s cannot install packages", m)
|
|
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
|
|
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
|
|
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
|
|
}
|
|
return o
|
|
}
|
|
|
|
// plainHow is how a machine failed, as the operator reads it.
|
|
func plainHow(d inventory.Doing) string {
|
|
if d.Outcome == inventory.OutcomeRefused {
|
|
return "refused every update"
|
|
}
|
|
return "applied its updates only in part"
|
|
}
|
|
|
|
// unansweringTool is the package manager a machine said does not answer, or "".
|
|
func unansweringTool(d inventory.Doing) string {
|
|
for _, r := range d.Failed {
|
|
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
|
|
return strings.TrimSuffix(rest, ")")
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
|
|
func failedNames(d inventory.Doing) string {
|
|
if d.Outcome == inventory.OutcomeRefused {
|
|
return "refused"
|
|
}
|
|
ids := make([]string, 0, len(d.Failed))
|
|
for _, r := range d.Failed {
|
|
ids = append(ids, r.ID)
|
|
}
|
|
sort.Strings(ids)
|
|
return "failed " + strings.Join(ids, ", ")
|
|
}
|
|
|
|
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
|
|
func doingWords(d inventory.Doing) string {
|
|
if d.Outcome == inventory.OutcomeRefused {
|
|
return "refused: " + oneLine(d.Refused)
|
|
}
|
|
var parts []string
|
|
for _, r := range d.Failed {
|
|
parts = append(parts, r.ID+": "+oneLine(r.Error))
|
|
}
|
|
sort.Strings(parts)
|
|
return strings.Join(parts, "; ")
|
|
}
|