|
|
|
@@ -0,0 +1,687 @@
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
|
|
|
|
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
|
|
|
|
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
|
|
|
|
//
|
|
|
|
|
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
|
|
|
|
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
|
|
|
|
// suite is held to having words, and to their being plain (plain_words_test.go).
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"fmt"
|
|
|
|
|
"sort"
|
|
|
|
|
"strings"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
|
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type words = conditions.Words
|
|
|
|
|
|
|
|
|
|
// worded is a wording that needs only the condition's subject.
|
|
|
|
|
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
|
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
|
for kind, fn := range plainWordings {
|
|
|
|
|
conditions.Wording(kind, fn)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// machineOr is the machine a condition concerns, or what to say when none is named.
|
|
|
|
|
func machineOr(o conditions.Observation, none string) string {
|
|
|
|
|
if o.Machine != "" {
|
|
|
|
|
return o.Machine
|
|
|
|
|
}
|
|
|
|
|
return none
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// idPart is the n-th dotted part of a condition's id, or "".
|
|
|
|
|
func idPart(o conditions.Observation, n int) string {
|
|
|
|
|
parts := strings.Split(o.ID, ".")
|
|
|
|
|
if n < len(parts) {
|
|
|
|
|
return parts[n]
|
|
|
|
|
}
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
|
|
|
|
func dataWords(o conditions.Observation) (what, module string) {
|
|
|
|
|
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
|
|
|
|
switch {
|
|
|
|
|
case module != "" && item != "":
|
|
|
|
|
return item + " of " + module, module
|
|
|
|
|
case module != "":
|
|
|
|
|
return "data of " + module, module
|
|
|
|
|
}
|
|
|
|
|
return "data", "a module"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var plainWordings = map[string]func(conditions.Observation) words{
|
|
|
|
|
// Machines and their node-engine.
|
|
|
|
|
"silent": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " is not answering",
|
|
|
|
|
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
|
|
|
|
"normal; if %s should be on, check that it is powered and online.", m, m),
|
|
|
|
|
Resolved: m + " answers again"}
|
|
|
|
|
}),
|
|
|
|
|
"sent-not-reported": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " has not confirmed a change",
|
|
|
|
|
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them. It "+
|
|
|
|
|
"usually catches up by itself; if it stays, %s may be stuck.", m, m, m),
|
|
|
|
|
Resolved: m + " applied the change"}
|
|
|
|
|
}),
|
|
|
|
|
"tools-silent": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "Tools on " + m + " are not answering",
|
|
|
|
|
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
|
|
|
|
"the mesh. A machine asleep is normal.", m, m),
|
|
|
|
|
Resolved: "Tools on " + m + " answer again"}
|
|
|
|
|
}),
|
|
|
|
|
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " waits for new instructions",
|
|
|
|
|
Explanation: fmt.Sprintf("%s's instructions only work once the next push is made, and none has been made "+
|
|
|
|
|
"for a while. Nothing is broken; an agent can push to %s.", conditions.Capital(m), m),
|
|
|
|
|
Resolved: m + " got its new instructions"}
|
|
|
|
|
}),
|
|
|
|
|
"declaration-refused": worded(func(o conditions.Observation) words {
|
|
|
|
|
if o.Scope == conditions.ScopeMesh {
|
|
|
|
|
return words{Headline: "No machine can get new instructions",
|
|
|
|
|
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
|
|
|
|
"made. Machines keep running what they have. It needs a look now.",
|
|
|
|
|
Resolved: "Machines can get new instructions again"}
|
|
|
|
|
}
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " cannot get new instructions",
|
|
|
|
|
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new reaches "+
|
|
|
|
|
"%s. It keeps running what it has. It needs a fix in the catalogue.", m, m),
|
|
|
|
|
Resolved: m + " can get new instructions again"}
|
|
|
|
|
}),
|
|
|
|
|
"own-address-banned": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " has banned the mesh",
|
|
|
|
|
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
|
|
|
|
"mesh is locked out of %s. It needs the ban lifted.", m, m),
|
|
|
|
|
Resolved: m + " no longer bans the mesh"}
|
|
|
|
|
}),
|
|
|
|
|
"core-behind": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " runs an old node-engine",
|
|
|
|
|
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds, and no "+
|
|
|
|
|
"delivery is bringing it up to date. It works, but misses the newest fixes.", conditions.Capital(m)),
|
|
|
|
|
Resolved: m + " runs the current node-engine"}
|
|
|
|
|
}),
|
|
|
|
|
kindMachineUnits: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "Failed services on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("Services on %s that no module manages have failed. They are %s's own, so the "+
|
|
|
|
|
"mesh does not repair them: mend or remove them there.", m, m),
|
|
|
|
|
Resolved: "No failed services on " + m + " any more"}
|
|
|
|
|
}),
|
|
|
|
|
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " has a network problem",
|
|
|
|
|
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
|
|
|
|
"route. Other machines may not reach it, or it them.", conditions.Capital(m)),
|
|
|
|
|
Resolved: m + "'s network is fine again"}
|
|
|
|
|
}),
|
|
|
|
|
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + "'s network settings were changed",
|
|
|
|
|
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
|
|
|
|
"client. Mesh names may not resolve there until it is undone.", m),
|
|
|
|
|
Resolved: m + "'s network settings are the mesh's again"}
|
|
|
|
|
}),
|
|
|
|
|
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: m + " cannot be reached",
|
|
|
|
|
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
|
|
|
|
"asleep is normal; otherwise its tunnel needs a look.", m),
|
|
|
|
|
Resolved: m + " can be reached again"}
|
|
|
|
|
}),
|
|
|
|
|
kindBindingKept: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "A module's data source is held on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
|
|
|
|
"kept the old one because switching would leave the data behind. It needs you to decide.", m),
|
|
|
|
|
Resolved: "Resolved: the data source on " + m + " is decided"}
|
|
|
|
|
}),
|
|
|
|
|
kindBindingMoving: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "A module's data source moves on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider, and its data is "+
|
|
|
|
|
"still with the old one. Move the data first.", m),
|
|
|
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
|
|
|
}),
|
|
|
|
|
kindBindingMoved: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "A module's data source moved on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("A module on %s switched to another provider of its data, and its data may "+
|
|
|
|
|
"still be with the old one. Check it found its data.", m),
|
|
|
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
|
|
|
}),
|
|
|
|
|
|
|
|
|
|
// Modules and providers.
|
|
|
|
|
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
|
|
|
|
thing := conditions.ThingWords(o)
|
|
|
|
|
return words{Headline: conditions.Capital(thing) + " is not working",
|
|
|
|
|
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again; if it stays, "+
|
|
|
|
|
"look at why it fails.", conditions.Capital(thing)),
|
|
|
|
|
Resolved: conditions.Capital(thing) + " works again"}
|
|
|
|
|
}),
|
|
|
|
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
|
|
|
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
|
|
|
|
if consumer == "" {
|
|
|
|
|
consumer = "a module"
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
|
|
|
|
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
|
|
|
|
"it up. %s may not work until it does.", conditions.Capital(thing), consumer, conditions.Capital(consumer)),
|
|
|
|
|
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
|
|
|
|
}),
|
|
|
|
|
kindProviderSilent: worded(func(o conditions.Observation) words {
|
|
|
|
|
thing := conditions.ThingWords(o)
|
|
|
|
|
return words{Headline: conditions.Capital(thing) + " went quiet",
|
|
|
|
|
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
|
|
|
|
"reporting. Whether it still fails is not known.", conditions.Capital(thing)),
|
|
|
|
|
Resolved: conditions.Capital(thing) + " reports again"}
|
|
|
|
|
}),
|
|
|
|
|
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
|
|
|
|
thing := conditions.ThingWords(o)
|
|
|
|
|
return words{Headline: "Retiring data on " + thing + " waits for you",
|
|
|
|
|
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own. It "+
|
|
|
|
|
"needs you to approve or reject retiring it.", conditions.Capital(thing)),
|
|
|
|
|
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
|
|
|
|
}),
|
|
|
|
|
kindRetireRejected: worded(func(o conditions.Observation) words {
|
|
|
|
|
thing := conditions.ThingWords(o)
|
|
|
|
|
return words{Headline: "Unused data kept on " + thing,
|
|
|
|
|
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for. Nothing to do; "+
|
|
|
|
|
"this is a reminder that it is still there.", thing),
|
|
|
|
|
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
|
|
|
|
}),
|
|
|
|
|
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
|
|
|
|
var where string
|
|
|
|
|
if o.Scope == conditions.ScopeProvider {
|
|
|
|
|
where = conditions.ThingWords(o)
|
|
|
|
|
} else {
|
|
|
|
|
what, _ := dataWords(o)
|
|
|
|
|
where = what + " on " + machineOr(o, "a machine")
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: "Retired data waits for cleanup",
|
|
|
|
|
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days. It needs you to delete "+
|
|
|
|
|
"it or keep it.", where),
|
|
|
|
|
Resolved: "Resolved: the retired data is dealt with"}
|
|
|
|
|
}),
|
|
|
|
|
|
|
|
|
|
// Data.
|
|
|
|
|
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "Data on " + m + " could not be measured",
|
|
|
|
|
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
|
|
|
|
"the data there: not that it is safe either.", m),
|
|
|
|
|
Resolved: "Data on " + m + " is measured again"}
|
|
|
|
|
}),
|
|
|
|
|
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
|
|
|
|
what, _ := dataWords(o)
|
|
|
|
|
return words{Headline: conditions.Capital(what) + " is unprotected",
|
|
|
|
|
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
|
|
|
|
"the mesh can see. It needs you to check its storage.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
|
|
|
Resolved: conditions.Capital(what) + " is protected again"}
|
|
|
|
|
}),
|
|
|
|
|
kindDataMissing: worded(func(o conditions.Observation) words {
|
|
|
|
|
what, _ := dataWords(o)
|
|
|
|
|
return words{Headline: conditions.Capital(what) + " is gone",
|
|
|
|
|
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more. If that was not meant, it needs "+
|
|
|
|
|
"restoring from a backup.", what, machineOr(o, "its machine")),
|
|
|
|
|
Resolved: conditions.Capital(what) + " is back"}
|
|
|
|
|
}),
|
|
|
|
|
kindDataShrank: worded(func(o conditions.Observation) words {
|
|
|
|
|
what, _ := dataWords(o)
|
|
|
|
|
m := machineOr(o, "its machine")
|
|
|
|
|
if idPart(o, 1) == "dataset" {
|
|
|
|
|
what = "a dataset"
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
|
|
|
|
Explanation: "More than half of what it held is gone within a week. If you meant that, silence it; if " +
|
|
|
|
|
"not, restore the last good copy from a backup.",
|
|
|
|
|
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
|
|
|
|
}),
|
|
|
|
|
kindDataQuiet: worded(func(o conditions.Observation) words {
|
|
|
|
|
what, _ := dataWords(o)
|
|
|
|
|
return words{Headline: conditions.Capital(what) + " stopped changing",
|
|
|
|
|
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
|
|
|
|
"usual: whatever writes it has stopped.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
|
|
|
Resolved: conditions.Capital(what) + " is written again"}
|
|
|
|
|
}),
|
|
|
|
|
kindBackupStale: worded(func(o conditions.Observation) words {
|
|
|
|
|
what, _ := dataWords(o)
|
|
|
|
|
return words{Headline: "No recent backup of " + what,
|
|
|
|
|
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
|
|
|
|
"changed since would be lost.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
|
|
|
Resolved: conditions.Capital(what) + " is backed up again"}
|
|
|
|
|
}),
|
|
|
|
|
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "Storage on " + m + " is degraded",
|
|
|
|
|
Explanation: fmt.Sprintf("The redundant storage on %s lost a disk or is rebuilding. The data on it is "+
|
|
|
|
|
"less protected until it is whole again.", m),
|
|
|
|
|
Resolved: "Storage on " + m + " is whole again"}
|
|
|
|
|
}),
|
|
|
|
|
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
|
|
|
|
module := idPart(o, 1)
|
|
|
|
|
if o.Scope == conditions.ScopeProvider {
|
|
|
|
|
module = idPart(o, 2)
|
|
|
|
|
}
|
|
|
|
|
if module == "" {
|
|
|
|
|
module = "a module"
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
|
|
|
|
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
|
|
|
|
"elsewhere. It needs you: move the data, or move it back.", conditions.Capital(module), machineOr(o, "its machine")),
|
|
|
|
|
Resolved: conditions.Capital(module) + " has its data again"}
|
|
|
|
|
}),
|
|
|
|
|
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
|
|
|
|
consumer := idPart(o, 1)
|
|
|
|
|
if consumer == "" {
|
|
|
|
|
consumer = "a module"
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
|
|
|
|
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one. It needs you to "+
|
|
|
|
|
"decide which to keep.", conditions.Capital(consumer)),
|
|
|
|
|
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
|
|
|
|
}),
|
|
|
|
|
|
|
|
|
|
// Deliveries and walks.
|
|
|
|
|
"stalled": worded(func(o conditions.Observation) words {
|
|
|
|
|
if o.Scope == conditions.ScopeDelivery {
|
|
|
|
|
return words{Headline: "A delivery is held too long",
|
|
|
|
|
Explanation: "A delivery has been held past its time limit. " + conditions.ResolverWords(o),
|
|
|
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: "A delivery is stuck halfway",
|
|
|
|
|
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost; if it " +
|
|
|
|
|
"stays, an agent can look at why.",
|
|
|
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
|
|
|
}),
|
|
|
|
|
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A delivery is waiting to start",
|
|
|
|
|
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
|
|
|
|
"out) has not said it may start. Nothing to do yet; an agent can start it by hand.",
|
|
|
|
|
Resolved: "Resolved: the delivery is no longer waiting"}
|
|
|
|
|
}),
|
|
|
|
|
"merge-not-acted": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A merge was never picked up",
|
|
|
|
|
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
|
|
|
|
"sent. An agent can hand it over again.",
|
|
|
|
|
Resolved: "Resolved: the merge was picked up"}
|
|
|
|
|
}),
|
|
|
|
|
"ask-lost": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A build has no result",
|
|
|
|
|
Explanation: "A build was asked for and never answered. What it was for is not delivered until it is " +
|
|
|
|
|
"built again.",
|
|
|
|
|
Resolved: "Resolved: the build has its result"}
|
|
|
|
|
}),
|
|
|
|
|
kindRolledBack: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
module := idPart(o, 0)
|
|
|
|
|
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
|
|
|
|
module = "a module"
|
|
|
|
|
}
|
|
|
|
|
if o.Scope == conditions.ScopeCore {
|
|
|
|
|
return words{Headline: "A core update was put back on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
|
|
|
|
"before was put back. It needs a look now.", module, m),
|
|
|
|
|
Resolved: "Resolved: the core on " + m + " is healthy"}
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
|
|
|
|
"before. The change is not delivered; it needs a fix in its code.", module, m),
|
|
|
|
|
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
|
|
|
|
}),
|
|
|
|
|
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "Putting back an update failed on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before. What runs "+
|
|
|
|
|
"there may be broken; it needs you now.", m),
|
|
|
|
|
Resolved: "Resolved: " + m + " runs a good build again"}
|
|
|
|
|
}),
|
|
|
|
|
"release-held": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Updates wait for your release",
|
|
|
|
|
Explanation: "Some module updates wait for a person to release them. They are not delivered until you do.",
|
|
|
|
|
Resolved: "Resolved: the held updates are released"}
|
|
|
|
|
}),
|
|
|
|
|
"facts-stale": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Merge checks use outdated facts",
|
|
|
|
|
Explanation: "The facts a merge check judges a change against are out of date, so a check may pass or " +
|
|
|
|
|
"fail for the wrong reason.",
|
|
|
|
|
Resolved: "Merge checks use current facts again"}
|
|
|
|
|
}),
|
|
|
|
|
|
|
|
|
|
// The controller and the core.
|
|
|
|
|
"controller-deaf": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The controller stopped listening",
|
|
|
|
|
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
|
|
|
|
"wait. Changes and repairs do not happen until it recovers.",
|
|
|
|
|
Resolved: "The controller listens again"}
|
|
|
|
|
}),
|
|
|
|
|
"self-check-silent": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The mesh's self-check stopped",
|
|
|
|
|
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
|
|
|
|
"Problems may go unnoticed until it runs again.",
|
|
|
|
|
Resolved: "The self-check runs again"}
|
|
|
|
|
}),
|
|
|
|
|
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The mesh's watchdogs stopped",
|
|
|
|
|
Explanation: "The watchdogs, which notice when something expected does not happen, have not run. Missed " +
|
|
|
|
|
"signals are not being noticed.",
|
|
|
|
|
Resolved: "The watchdogs run again"}
|
|
|
|
|
}),
|
|
|
|
|
"lease-lost": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The controller lost its lease",
|
|
|
|
|
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
|
|
|
|
"not renew it, so two could act at once, or none. It needs a look now.",
|
|
|
|
|
Resolved: "The controller holds its lease again"}
|
|
|
|
|
}),
|
|
|
|
|
"lease-split": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Two controllers may be acting",
|
|
|
|
|
Explanation: "The lease that lets one controller act at a time does not agree with its record, so two " +
|
|
|
|
|
"could act at once. It needs a look now.",
|
|
|
|
|
Resolved: "One controller acts again"}
|
|
|
|
|
}),
|
|
|
|
|
"stale-writer": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Outdated instructions are being sent",
|
|
|
|
|
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
|
|
|
|
"were refused. It is harmless if it stops.",
|
|
|
|
|
Resolved: "Resolved: no more outdated instructions"}
|
|
|
|
|
}),
|
|
|
|
|
"status-slow": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The mesh answers slowly",
|
|
|
|
|
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
|
|
|
|
Resolved: "The mesh answers quickly again"}
|
|
|
|
|
}),
|
|
|
|
|
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
|
|
|
|
if o.Scope == conditions.ScopeBus {
|
|
|
|
|
return words{Headline: "The bus is not healthy",
|
|
|
|
|
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
|
|
|
|
"and tool calls may fail until it recovers.",
|
|
|
|
|
Resolved: "The bus is healthy again"}
|
|
|
|
|
}
|
|
|
|
|
m := machineOr(o, "its machine")
|
|
|
|
|
return words{Headline: "The mesh's own software fails on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy. What it does there may "+
|
|
|
|
|
"not happen until it recovers.", m),
|
|
|
|
|
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
|
|
|
|
}),
|
|
|
|
|
"call-hung": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A tool call is hanging",
|
|
|
|
|
Explanation: "A call to one of the mesh's tools has run far past its time limit. Whatever waits on it is " +
|
|
|
|
|
"stuck.",
|
|
|
|
|
Resolved: "Resolved: the tool call ended"}
|
|
|
|
|
}),
|
|
|
|
|
"probe-failed": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A check of the mesh could not run",
|
|
|
|
|
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown, which is not " +
|
|
|
|
|
"the same as fine.",
|
|
|
|
|
Resolved: "The check runs again"}
|
|
|
|
|
}),
|
|
|
|
|
kindHealersBraked: worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "All automatic repairs stopped",
|
|
|
|
|
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
|
|
|
|
"looks like a loop, so all of them stopped. Nothing repairs itself until this is looked at.",
|
|
|
|
|
Resolved: "Automatic repairs run again"}
|
|
|
|
|
}),
|
|
|
|
|
"healer-wanted": worded(func(o conditions.Observation) words {
|
|
|
|
|
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
|
|
|
|
if o.Scope != conditions.ScopeMesh {
|
|
|
|
|
cause = "something"
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
|
|
|
|
Explanation: "A person keeps repairing this by hand. A healer (an automatic repair) is wanted for it; " +
|
|
|
|
|
"nothing is broken now.",
|
|
|
|
|
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
|
|
|
|
}),
|
|
|
|
|
|
|
|
|
|
// The bus and seats.
|
|
|
|
|
"resolver-wrong": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
return words{Headline: "Name lookups fail on " + m,
|
|
|
|
|
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly. "+
|
|
|
|
|
"Machines that rely on it may not find each other.", m),
|
|
|
|
|
Resolved: "Name lookups work on " + m + " again"}
|
|
|
|
|
}),
|
|
|
|
|
"holder-silent": worded(func(o conditions.Observation) words {
|
|
|
|
|
m := machineOr(o, "a machine")
|
|
|
|
|
seat := idPart(o, 0)
|
|
|
|
|
if o.Scope != conditions.ScopeSeat {
|
|
|
|
|
seat = "a seat"
|
|
|
|
|
}
|
|
|
|
|
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
|
|
|
|
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
|
|
|
|
"machine asleep is normal.", seat, m),
|
|
|
|
|
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
|
|
|
|
}),
|
|
|
|
|
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The bus could not be fully set up",
|
|
|
|
|
Explanation: "When instructions were sent, the bus's queues could not all be checked, so a module may " +
|
|
|
|
|
"not get its messages.",
|
|
|
|
|
Resolved: "The bus is fully set up again"}
|
|
|
|
|
}),
|
|
|
|
|
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The bus is being upgraded",
|
|
|
|
|
Explanation: "The mesh's message system is being upgraded; some things pause until it is done. Nothing " +
|
|
|
|
|
"to do.",
|
|
|
|
|
Resolved: "The bus upgrade is done"}
|
|
|
|
|
}),
|
|
|
|
|
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The bus upgrade failed",
|
|
|
|
|
Explanation: "The upgrade of the mesh's message system did not end healthy in time. It needs you: it can " +
|
|
|
|
|
"be put back to the version before.",
|
|
|
|
|
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
|
|
|
|
}),
|
|
|
|
|
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Messages pile up for a listener",
|
|
|
|
|
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
|
|
|
|
"healer may restart it.",
|
|
|
|
|
Resolved: "The listener caught up"}
|
|
|
|
|
}),
|
|
|
|
|
"consumer-wrong": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A listener on the bus is set up wrong",
|
|
|
|
|
Explanation: "One of the mesh's listeners on the bus is missing or not as the controller defines it, so " +
|
|
|
|
|
"some messages may reach nobody.",
|
|
|
|
|
Resolved: "The listener is set up right again"}
|
|
|
|
|
}),
|
|
|
|
|
"consumer-lost": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A listener on the bus is gone",
|
|
|
|
|
Explanation: "One of the mesh's listeners is no longer on the bus, so what it should receive reaches " +
|
|
|
|
|
"nobody.",
|
|
|
|
|
Resolved: "The listener is back"}
|
|
|
|
|
}),
|
|
|
|
|
"slow-consumer": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A listener on the bus is too slow",
|
|
|
|
|
Explanation: "The bus reports a listener too slow to keep up, so messages to it are dropped or late.",
|
|
|
|
|
Resolved: "The listener keeps up again"}
|
|
|
|
|
}),
|
|
|
|
|
"max-deliveries": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "A message could not be handled",
|
|
|
|
|
Explanation: "The bus gave up on a message after trying to hand it over too many times. What it carried " +
|
|
|
|
|
"did not happen.",
|
|
|
|
|
Resolved: "Resolved: messages are handled again"}
|
|
|
|
|
}),
|
|
|
|
|
"refused": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "The bus refuses some messages",
|
|
|
|
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
|
|
|
|
Resolved: "Resolved: the bus takes the messages again"}
|
|
|
|
|
}),
|
|
|
|
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Part of the bus's storage is wrong",
|
|
|
|
|
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps " +
|
|
|
|
|
"there may be lost.",
|
|
|
|
|
Resolved: "The bus's storage is right again"}
|
|
|
|
|
}),
|
|
|
|
|
"archives-unheld": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Kept builds are about to be cleaned up",
|
|
|
|
|
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them. " +
|
|
|
|
|
"Nothing to do unless one is still needed.",
|
|
|
|
|
Resolved: "Resolved: every kept build is claimed"}
|
|
|
|
|
}),
|
|
|
|
|
"archives-missing": worded(func(o conditions.Observation) words {
|
|
|
|
|
return words{Headline: "Kept builds are missing",
|
|
|
|
|
Explanation: "Some builds the mesh relies on are no longer in the store. A machine that needs one cannot " +
|
|
|
|
|
"get it until it is built again.",
|
|
|
|
|
Resolved: "Resolved: the kept builds are back"}
|
|
|
|
|
}),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
|
|
|
|
func humanDuration(d time.Duration) string {
|
|
|
|
|
switch {
|
|
|
|
|
case d < 2*time.Minute:
|
|
|
|
|
return "a minute"
|
|
|
|
|
case d < 2*time.Hour:
|
|
|
|
|
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
|
|
|
|
case d < 48*time.Hour:
|
|
|
|
|
return fmt.Sprintf("%d hours", int(d.Hours()))
|
|
|
|
|
}
|
|
|
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// repoName is a repository as a person says it: its name without its owner.
|
|
|
|
|
func repoName(repository string) string {
|
|
|
|
|
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
|
|
|
|
return repository[i+1:]
|
|
|
|
|
}
|
|
|
|
|
return repository
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
|
|
|
|
func namesWords(names []string, most int) string {
|
|
|
|
|
switch {
|
|
|
|
|
case len(names) == 0:
|
|
|
|
|
return ""
|
|
|
|
|
case len(names) == 1:
|
|
|
|
|
return names[0]
|
|
|
|
|
case len(names) <= most:
|
|
|
|
|
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
|
|
|
|
}
|
|
|
|
|
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// planModules are the modules a walk moves, sorted.
|
|
|
|
|
func planModules(p inventory.Plan) []string {
|
|
|
|
|
out := make([]string, 0, len(p.Modules))
|
|
|
|
|
for m := range p.Modules {
|
|
|
|
|
out = append(out, m)
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(out)
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
|
|
|
|
// their count and the repository.
|
|
|
|
|
func deliveryWhat(modules []string, repository string) string {
|
|
|
|
|
if len(modules) > 0 && len(modules) <= 2 {
|
|
|
|
|
return namesWords(modules, 2)
|
|
|
|
|
}
|
|
|
|
|
if len(modules) > 2 {
|
|
|
|
|
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
|
|
|
|
}
|
|
|
|
|
return repoName(repository)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
|
|
|
|
func deliveryName(modules []string, repository string) string {
|
|
|
|
|
if len(modules) > 0 && len(modules) <= 2 {
|
|
|
|
|
return namesWords(modules, 2) + " delivery"
|
|
|
|
|
}
|
|
|
|
|
return repoName(repository) + " delivery"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// waitingAct is what the operator does about a walk waiting for its delivery's word.
|
|
|
|
|
func waitingAct(severity conditions.Severity) string {
|
|
|
|
|
if severity == conditions.Urgent {
|
|
|
|
|
return "Mesh-delivery may be broken. It needs you: have an agent look at mesh-delivery, or start the " +
|
|
|
|
|
"delivery by hand."
|
|
|
|
|
}
|
|
|
|
|
return fmt.Sprintf("Nothing to do yet; it becomes urgent after %s. To start it now, have an agent start it by hand.",
|
|
|
|
|
humanDuration(waitUrgentAfter))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
|
|
|
|
func stalledWords(l stalledLine, o conditions.Observation) (string, string, string) {
|
|
|
|
|
repository, _, _ := strings.Cut(l.ID, "@")
|
|
|
|
|
name := repoName(repository)
|
|
|
|
|
held := l.State
|
|
|
|
|
if held == "" {
|
|
|
|
|
held = "held"
|
|
|
|
|
}
|
|
|
|
|
long := "too long"
|
|
|
|
|
if d, err := time.ParseDuration(l.For); err == nil {
|
|
|
|
|
long = "for " + humanDuration(d)
|
|
|
|
|
}
|
|
|
|
|
act := conditions.ResolverWords(o)
|
|
|
|
|
if o.Resolver == conditions.ResolverOperator {
|
|
|
|
|
act = "It waits for you: release it, or stop it."
|
|
|
|
|
}
|
|
|
|
|
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
|
|
|
|
fmt.Sprintf("A delivery of %s has been %s %s, past its limit. %s", name, held, long, act),
|
|
|
|
|
fmt.Sprintf("Delivery of %s is no longer %s", name, held)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// causeWords is a hand-act's cause as a person says it.
|
|
|
|
|
func causeWords(cause string) string {
|
|
|
|
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
|
|
|
|
func unitPlainWords(unit string) string {
|
|
|
|
|
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
|
|
|
|
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
|
|
|
|
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
|
|
|
|
if what == "" {
|
|
|
|
|
return strings.ReplaceAll(name, ".", " ")
|
|
|
|
|
}
|
|
|
|
|
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return strings.ReplaceAll(unit, ".", " ")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
|
|
|
|
func resourcePlainWords(r inventory.ResourceHealth) string {
|
|
|
|
|
if r.Kind == link.KindUnit {
|
|
|
|
|
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
|
|
|
|
}
|
|
|
|
|
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
|
|
|
|
switch r.Reason {
|
|
|
|
|
case "restarting":
|
|
|
|
|
return what + " keeps restarting"
|
|
|
|
|
case "down":
|
|
|
|
|
return what + " is not running"
|
|
|
|
|
}
|
|
|
|
|
if r.Check != "" {
|
|
|
|
|
return what + " fails its " + r.Check + " check"
|
|
|
|
|
}
|
|
|
|
|
return what + " is not healthy"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
|
|
|
|
// systemd's result, such as exit-code, or a sentence carrying it).
|
|
|
|
|
func unitReasonWords(reason string) string {
|
|
|
|
|
switch {
|
|
|
|
|
case strings.Contains(reason, "start-limit"):
|
|
|
|
|
return "failed too often to be started again"
|
|
|
|
|
case strings.Contains(reason, "exit-code"):
|
|
|
|
|
return "stopped with an error"
|
|
|
|
|
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
|
|
|
|
return "crashed"
|
|
|
|
|
case strings.Contains(reason, "timeout"):
|
|
|
|
|
return "timed out"
|
|
|
|
|
}
|
|
|
|
|
return "failed"
|
|
|
|
|
}
|