package main import ( "fmt" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) // kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part, // for longer than one apply interval (novox/hq issue 233). const kindApplyFailing = "apply-failing" // applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its // ReconcileEvery. A failure still there after one more of them is not one the next apply clears. const applyEvery = 5 * time.Minute // packageManagerKey is how a node-engine names a package manager that does not answer among what failed // (mesh-host's probe.go): "the package manager (pacman)". const packageManagerKey = "the package manager (" // watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only // to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every // declaration for an hour and a half because its package manager did not answer, and the mesh's only word // on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply // interval **while the machine is heard** โ€” judged by time, not by how many reports said it: a node-engine // says a re-apply only when how it went changed, so the same failure is one report however long it lasts. // Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while // the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. func watchApplyFailing(f *signalFacts) []conditions.Observation { heard := map[string]bool{} for _, m := range f.machines { if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) { heard[m.name] = true } } var out []conditions.Observation said := map[string]bool{} for _, d := range f.doing { if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { continue } out = append(out, applyFailing(d, f.now)) said[d.Node] = true } // **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 ยง3): its client's // newest word, judged by the same bound โ€” the client says a failure once, and fills again on every change โ€” // on a machine whose node-engine applied what it was sent. failed := map[string][]inventory.ProvReport{} for _, r := range f.provFailed { if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] { continue } failed[r.Node] = append(failed[r.Node], r) } for _, node := range sortedNames(failed) { out = append(out, provisioningFailing(node, failed[node], f.now)) } return out } // provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to // place. func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation { sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module }) var names, words []string since := failed[0].At for _, r := range failed { names = append(names, r.Module) words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason))) if r.At.Before(since) { since = r.At } } o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, Severity: conditions.Warning, Said: strings.Join(words, "; "), Resolved: m + " applies what it is sent again"} o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m, strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery)) o.Headline = m + " is not applying its update" o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+ "details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since))) return o } func orNoReason(s string) string { if s == "" { return "no reason given" } return s } // applyFailing is one machine's S21 condition. func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { m := d.Node since := d.Since.UTC().Format("2006-01-02 15:04 MST") o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, Severity: conditions.Warning, Said: doingWords(d), Resolved: m + " applies what it is sent again"} how := "refused its declaration whole" if d.Outcome != inventory.OutcomeRefused { how = "applied its declaration only in part" } o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d)) o.Headline = m + " is not applying its update" o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", m, plainHow(d), humanDuration(now.Sub(*d.Since))) if tool := unansweringTool(d); tool != "" { o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ "is applied, since %s (bound %s)", m, tool, since, ago(applyEvery)) o.Headline = fmt.Sprintf("%s cannot install packages", m) o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ "look on the machine.", m, tool, humanDuration(now.Sub(*d.Since))) } return o } // plainHow is how a machine failed, as the operator reads it. func plainHow(d inventory.Doing) string { if d.Outcome == inventory.OutcomeRefused { return "refused every update" } return "applied its updates only in part" } // unansweringTool is the package manager a machine said does not answer, or "". func unansweringTool(d inventory.Doing) string { for _, r := range d.Failed { if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok { return strings.TrimSuffix(rest, ")") } } return "" } // failedNames is what failed, by name: the refusal, or the failed resources' ids. func failedNames(d inventory.Doing) string { if d.Outcome == inventory.OutcomeRefused { return "refused" } ids := make([]string, 0, len(d.Failed)) for _, r := range d.Failed { ids = append(ids, r.ID) } sort.Strings(ids) return "failed " + strings.Join(ids, ", ") } // doingWords is the machine's own words on what failed: the evidence, kept inside the mesh. func doingWords(d inventory.Doing) string { if d.Outcome == inventory.OutcomeRefused { return "refused: " + oneLine(d.Refused) } var parts []string for _, r := range d.Failed { parts = append(parts, r.ID+": "+oneLine(r.Error)) } sort.Strings(parts) return strings.Join(parts, "; ") }