Give every condition a headline, an explanation and a resolved line in plain words
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: --- FAIL: TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt (0.00s)
mesh/delivery-group group feat/plain-notifications rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: --- FAIL: TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt (0.00s)
mesh/delivery-group group feat/plain-notifications rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
The operator could not read the desktop notifications: they carried plan ids, commits, keys and verb syntax. The words the operator reads now travel with the condition, so every channel says them (hq ADR 0253).
This commit is contained in:
jochen
@@ -0,0 +1,687 @@
|
||||
package main
|
||||
|
||||
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
||||
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
||||
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
||||
//
|
||||
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
||||
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
||||
// suite is held to having words, and to their being plain (plain_words_test.go).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
type words = conditions.Words
|
||||
|
||||
// worded is a wording that needs only the condition's subject.
|
||||
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
||||
|
||||
func init() {
|
||||
for kind, fn := range plainWordings {
|
||||
conditions.Wording(kind, fn)
|
||||
}
|
||||
}
|
||||
|
||||
// machineOr is the machine a condition concerns, or what to say when none is named.
|
||||
func machineOr(o conditions.Observation, none string) string {
|
||||
if o.Machine != "" {
|
||||
return o.Machine
|
||||
}
|
||||
return none
|
||||
}
|
||||
|
||||
// idPart is the n-th dotted part of a condition's id, or "".
|
||||
func idPart(o conditions.Observation, n int) string {
|
||||
parts := strings.Split(o.ID, ".")
|
||||
if n < len(parts) {
|
||||
return parts[n]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
||||
func dataWords(o conditions.Observation) (what, module string) {
|
||||
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
||||
switch {
|
||||
case module != "" && item != "":
|
||||
return item + " of " + module, module
|
||||
case module != "":
|
||||
return "data of " + module, module
|
||||
}
|
||||
return "data", "a module"
|
||||
}
|
||||
|
||||
var plainWordings = map[string]func(conditions.Observation) words{
|
||||
// Machines and their node-engine.
|
||||
"silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " is not answering",
|
||||
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
||||
"normal; if %s should be on, check that it is powered and online.", m, m),
|
||||
Resolved: m + " answers again"}
|
||||
}),
|
||||
"sent-not-reported": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has not confirmed a change",
|
||||
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them. It "+
|
||||
"usually catches up by itself; if it stays, %s may be stuck.", m, m, m),
|
||||
Resolved: m + " applied the change"}
|
||||
}),
|
||||
"tools-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Tools on " + m + " are not answering",
|
||||
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
||||
"the mesh. A machine asleep is normal.", m, m),
|
||||
Resolved: "Tools on " + m + " answer again"}
|
||||
}),
|
||||
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " waits for new instructions",
|
||||
Explanation: fmt.Sprintf("%s's instructions only work once the next push is made, and none has been made "+
|
||||
"for a while. Nothing is broken; an agent can push to %s.", conditions.Capital(m), m),
|
||||
Resolved: m + " got its new instructions"}
|
||||
}),
|
||||
"declaration-refused": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeMesh {
|
||||
return words{Headline: "No machine can get new instructions",
|
||||
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
||||
"made. Machines keep running what they have. It needs a look now.",
|
||||
Resolved: "Machines can get new instructions again"}
|
||||
}
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot get new instructions",
|
||||
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new reaches "+
|
||||
"%s. It keeps running what it has. It needs a fix in the catalogue.", m, m),
|
||||
Resolved: m + " can get new instructions again"}
|
||||
}),
|
||||
"own-address-banned": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has banned the mesh",
|
||||
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
||||
"mesh is locked out of %s. It needs the ban lifted.", m, m),
|
||||
Resolved: m + " no longer bans the mesh"}
|
||||
}),
|
||||
"core-behind": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " runs an old node-engine",
|
||||
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds, and no "+
|
||||
"delivery is bringing it up to date. It works, but misses the newest fixes.", conditions.Capital(m)),
|
||||
Resolved: m + " runs the current node-engine"}
|
||||
}),
|
||||
kindMachineUnits: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Failed services on " + m,
|
||||
Explanation: fmt.Sprintf("Services on %s that no module manages have failed. They are %s's own, so the "+
|
||||
"mesh does not repair them: mend or remove them there.", m, m),
|
||||
Resolved: "No failed services on " + m + " any more"}
|
||||
}),
|
||||
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has a network problem",
|
||||
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
||||
"route. Other machines may not reach it, or it them.", conditions.Capital(m)),
|
||||
Resolved: m + "'s network is fine again"}
|
||||
}),
|
||||
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + "'s network settings were changed",
|
||||
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
||||
"client. Mesh names may not resolve there until it is undone.", m),
|
||||
Resolved: m + "'s network settings are the mesh's again"}
|
||||
}),
|
||||
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot be reached",
|
||||
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
||||
"asleep is normal; otherwise its tunnel needs a look.", m),
|
||||
Resolved: m + " can be reached again"}
|
||||
}),
|
||||
kindBindingKept: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source is held on " + m,
|
||||
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
||||
"kept the old one because switching would leave the data behind. It needs you to decide.", m),
|
||||
Resolved: "Resolved: the data source on " + m + " is decided"}
|
||||
}),
|
||||
kindBindingMoving: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moves on " + m,
|
||||
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider, and its data is "+
|
||||
"still with the old one. Move the data first.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
kindBindingMoved: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moved on " + m,
|
||||
Explanation: fmt.Sprintf("A module on %s switched to another provider of its data, and its data may "+
|
||||
"still be with the old one. Check it found its data.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
|
||||
// Modules and providers.
|
||||
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " is not working",
|
||||
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again; if it stays, "+
|
||||
"look at why it fails.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " works again"}
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
||||
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
||||
"it up. %s may not work until it does.", conditions.Capital(thing), consumer, conditions.Capital(consumer)),
|
||||
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
||||
}),
|
||||
kindProviderSilent: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " went quiet",
|
||||
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
||||
"reporting. Whether it still fails is not known.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " reports again"}
|
||||
}),
|
||||
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Retiring data on " + thing + " waits for you",
|
||||
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own. It "+
|
||||
"needs you to approve or reject retiring it.", conditions.Capital(thing)),
|
||||
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
||||
}),
|
||||
kindRetireRejected: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Unused data kept on " + thing,
|
||||
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for. Nothing to do; "+
|
||||
"this is a reminder that it is still there.", thing),
|
||||
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
||||
}),
|
||||
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
||||
var where string
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
where = conditions.ThingWords(o)
|
||||
} else {
|
||||
what, _ := dataWords(o)
|
||||
where = what + " on " + machineOr(o, "a machine")
|
||||
}
|
||||
return words{Headline: "Retired data waits for cleanup",
|
||||
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days. It needs you to delete "+
|
||||
"it or keep it.", where),
|
||||
Resolved: "Resolved: the retired data is dealt with"}
|
||||
}),
|
||||
|
||||
// Data.
|
||||
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Data on " + m + " could not be measured",
|
||||
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
||||
"the data there: not that it is safe either.", m),
|
||||
Resolved: "Data on " + m + " is measured again"}
|
||||
}),
|
||||
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is unprotected",
|
||||
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
||||
"the mesh can see. It needs you to check its storage.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is protected again"}
|
||||
}),
|
||||
kindDataMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is gone",
|
||||
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more. If that was not meant, it needs "+
|
||||
"restoring from a backup.", what, machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is back"}
|
||||
}),
|
||||
kindDataShrank: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
m := machineOr(o, "its machine")
|
||||
if idPart(o, 1) == "dataset" {
|
||||
what = "a dataset"
|
||||
}
|
||||
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
||||
Explanation: "More than half of what it held is gone within a week. If you meant that, silence it; if " +
|
||||
"not, restore the last good copy from a backup.",
|
||||
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
||||
}),
|
||||
kindDataQuiet: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " stopped changing",
|
||||
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
||||
"usual: whatever writes it has stopped.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is written again"}
|
||||
}),
|
||||
kindBackupStale: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: "No recent backup of " + what,
|
||||
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
||||
"changed since would be lost.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is backed up again"}
|
||||
}),
|
||||
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Storage on " + m + " is degraded",
|
||||
Explanation: fmt.Sprintf("The redundant storage on %s lost a disk or is rebuilding. The data on it is "+
|
||||
"less protected until it is whole again.", m),
|
||||
Resolved: "Storage on " + m + " is whole again"}
|
||||
}),
|
||||
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
||||
module := idPart(o, 1)
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
module = idPart(o, 2)
|
||||
}
|
||||
if module == "" {
|
||||
module = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
||||
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
||||
"elsewhere. It needs you: move the data, or move it back.", conditions.Capital(module), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(module) + " has its data again"}
|
||||
}),
|
||||
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
||||
consumer := idPart(o, 1)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
||||
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one. It needs you to "+
|
||||
"decide which to keep.", conditions.Capital(consumer)),
|
||||
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
||||
}),
|
||||
|
||||
// Deliveries and walks.
|
||||
"stalled": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeDelivery {
|
||||
return words{Headline: "A delivery is held too long",
|
||||
Explanation: "A delivery has been held past its time limit. " + conditions.ResolverWords(o),
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}
|
||||
return words{Headline: "A delivery is stuck halfway",
|
||||
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost; if it " +
|
||||
"stays, an agent can look at why.",
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}),
|
||||
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery is waiting to start",
|
||||
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
||||
"out) has not said it may start. Nothing to do yet; an agent can start it by hand.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
"merge-not-acted": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A merge was never picked up",
|
||||
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
||||
"sent. An agent can hand it over again.",
|
||||
Resolved: "Resolved: the merge was picked up"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. What it was for is not delivered until it is " +
|
||||
"built again.",
|
||||
Resolved: "Resolved: the build has its result"}
|
||||
}),
|
||||
kindRolledBack: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
module := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
||||
module = "a module"
|
||||
}
|
||||
if o.Scope == conditions.ScopeCore {
|
||||
return words{Headline: "A core update was put back on " + m,
|
||||
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
||||
"before was put back. It needs a look now.", module, m),
|
||||
Resolved: "Resolved: the core on " + m + " is healthy"}
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
||||
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
||||
"before. The change is not delivered; it needs a fix in its code.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Putting back an update failed on " + m,
|
||||
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before. What runs "+
|
||||
"there may be broken; it needs you now.", m),
|
||||
Resolved: "Resolved: " + m + " runs a good build again"}
|
||||
}),
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Updates wait for your release",
|
||||
Explanation: "Some module updates wait for a person to release them. They are not delivered until you do.",
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date, so a check may pass or " +
|
||||
"fail for the wrong reason.",
|
||||
Resolved: "Merge checks use current facts again"}
|
||||
}),
|
||||
|
||||
// The controller and the core.
|
||||
"controller-deaf": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller stopped listening",
|
||||
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
||||
"wait. Changes and repairs do not happen until it recovers.",
|
||||
Resolved: "The controller listens again"}
|
||||
}),
|
||||
"self-check-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's self-check stopped",
|
||||
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
||||
"Problems may go unnoticed until it runs again.",
|
||||
Resolved: "The self-check runs again"}
|
||||
}),
|
||||
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's watchdogs stopped",
|
||||
Explanation: "The watchdogs, which notice when something expected does not happen, have not run. Missed " +
|
||||
"signals are not being noticed.",
|
||||
Resolved: "The watchdogs run again"}
|
||||
}),
|
||||
"lease-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller lost its lease",
|
||||
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
||||
"not renew it, so two could act at once, or none. It needs a look now.",
|
||||
Resolved: "The controller holds its lease again"}
|
||||
}),
|
||||
"lease-split": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Two controllers may be acting",
|
||||
Explanation: "The lease that lets one controller act at a time does not agree with its record, so two " +
|
||||
"could act at once. It needs a look now.",
|
||||
Resolved: "One controller acts again"}
|
||||
}),
|
||||
"stale-writer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Outdated instructions are being sent",
|
||||
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
||||
"were refused. It is harmless if it stops.",
|
||||
Resolved: "Resolved: no more outdated instructions"}
|
||||
}),
|
||||
"status-slow": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh answers slowly",
|
||||
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
||||
Resolved: "The mesh answers quickly again"}
|
||||
}),
|
||||
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeBus {
|
||||
return words{Headline: "The bus is not healthy",
|
||||
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
||||
"and tool calls may fail until it recovers.",
|
||||
Resolved: "The bus is healthy again"}
|
||||
}
|
||||
m := machineOr(o, "its machine")
|
||||
return words{Headline: "The mesh's own software fails on " + m,
|
||||
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy. What it does there may "+
|
||||
"not happen until it recovers.", m),
|
||||
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
||||
}),
|
||||
"call-hung": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A tool call is hanging",
|
||||
Explanation: "A call to one of the mesh's tools has run far past its time limit. Whatever waits on it is " +
|
||||
"stuck.",
|
||||
Resolved: "Resolved: the tool call ended"}
|
||||
}),
|
||||
"probe-failed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A check of the mesh could not run",
|
||||
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown, which is not " +
|
||||
"the same as fine.",
|
||||
Resolved: "The check runs again"}
|
||||
}),
|
||||
kindHealersBraked: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "All automatic repairs stopped",
|
||||
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
||||
"looks like a loop, so all of them stopped. Nothing repairs itself until this is looked at.",
|
||||
Resolved: "Automatic repairs run again"}
|
||||
}),
|
||||
"healer-wanted": worded(func(o conditions.Observation) words {
|
||||
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
||||
if o.Scope != conditions.ScopeMesh {
|
||||
cause = "something"
|
||||
}
|
||||
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
||||
Explanation: "A person keeps repairing this by hand. A healer (an automatic repair) is wanted for it; " +
|
||||
"nothing is broken now.",
|
||||
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
||||
}),
|
||||
|
||||
// The bus and seats.
|
||||
"resolver-wrong": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Name lookups fail on " + m,
|
||||
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly. "+
|
||||
"Machines that rely on it may not find each other.", m),
|
||||
Resolved: "Name lookups work on " + m + " again"}
|
||||
}),
|
||||
"holder-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
seat := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeSeat {
|
||||
seat = "a seat"
|
||||
}
|
||||
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
||||
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
||||
"machine asleep is normal.", seat, m),
|
||||
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
||||
}),
|
||||
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus could not be fully set up",
|
||||
Explanation: "When instructions were sent, the bus's queues could not all be checked, so a module may " +
|
||||
"not get its messages.",
|
||||
Resolved: "The bus is fully set up again"}
|
||||
}),
|
||||
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus is being upgraded",
|
||||
Explanation: "The mesh's message system is being upgraded; some things pause until it is done. Nothing " +
|
||||
"to do.",
|
||||
Resolved: "The bus upgrade is done"}
|
||||
}),
|
||||
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus upgrade failed",
|
||||
Explanation: "The upgrade of the mesh's message system did not end healthy in time. It needs you: it can " +
|
||||
"be put back to the version before.",
|
||||
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
"healer may restart it.",
|
||||
Resolved: "The listener caught up"}
|
||||
}),
|
||||
"consumer-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is set up wrong",
|
||||
Explanation: "One of the mesh's listeners on the bus is missing or not as the controller defines it, so " +
|
||||
"some messages may reach nobody.",
|
||||
Resolved: "The listener is set up right again"}
|
||||
}),
|
||||
"consumer-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is gone",
|
||||
Explanation: "One of the mesh's listeners is no longer on the bus, so what it should receive reaches " +
|
||||
"nobody.",
|
||||
Resolved: "The listener is back"}
|
||||
}),
|
||||
"slow-consumer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is too slow",
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are dropped or late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
"max-deliveries": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A message could not be handled",
|
||||
Explanation: "The bus gave up on a message after trying to hand it over too many times. What it carried " +
|
||||
"did not happen.",
|
||||
Resolved: "Resolved: messages are handled again"}
|
||||
}),
|
||||
"refused": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus refuses some messages",
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps " +
|
||||
"there may be lost.",
|
||||
Resolved: "The bus's storage is right again"}
|
||||
}),
|
||||
"archives-unheld": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are about to be cleaned up",
|
||||
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them. " +
|
||||
"Nothing to do unless one is still needed.",
|
||||
Resolved: "Resolved: every kept build is claimed"}
|
||||
}),
|
||||
"archives-missing": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are missing",
|
||||
Explanation: "Some builds the mesh relies on are no longer in the store. A machine that needs one cannot " +
|
||||
"get it until it is built again.",
|
||||
Resolved: "Resolved: the kept builds are back"}
|
||||
}),
|
||||
}
|
||||
|
||||
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
||||
func humanDuration(d time.Duration) string {
|
||||
switch {
|
||||
case d < 2*time.Minute:
|
||||
return "a minute"
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
// repoName is a repository as a person says it: its name without its owner.
|
||||
func repoName(repository string) string {
|
||||
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
||||
return repository[i+1:]
|
||||
}
|
||||
return repository
|
||||
}
|
||||
|
||||
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
||||
func namesWords(names []string, most int) string {
|
||||
switch {
|
||||
case len(names) == 0:
|
||||
return ""
|
||||
case len(names) == 1:
|
||||
return names[0]
|
||||
case len(names) <= most:
|
||||
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||
}
|
||||
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
||||
}
|
||||
|
||||
// planModules are the modules a walk moves, sorted.
|
||||
func planModules(p inventory.Plan) []string {
|
||||
out := make([]string, 0, len(p.Modules))
|
||||
for m := range p.Modules {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
||||
// their count and the repository.
|
||||
func deliveryWhat(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2)
|
||||
}
|
||||
if len(modules) > 2 {
|
||||
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
||||
}
|
||||
return repoName(repository)
|
||||
}
|
||||
|
||||
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
||||
func deliveryName(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2) + " delivery"
|
||||
}
|
||||
return repoName(repository) + " delivery"
|
||||
}
|
||||
|
||||
// waitingAct is what the operator does about a walk waiting for its delivery's word.
|
||||
func waitingAct(severity conditions.Severity) string {
|
||||
if severity == conditions.Urgent {
|
||||
return "Mesh-delivery may be broken. It needs you: have an agent look at mesh-delivery, or start the " +
|
||||
"delivery by hand."
|
||||
}
|
||||
return fmt.Sprintf("Nothing to do yet; it becomes urgent after %s. To start it now, have an agent start it by hand.",
|
||||
humanDuration(waitUrgentAfter))
|
||||
}
|
||||
|
||||
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
||||
func stalledWords(l stalledLine, o conditions.Observation) (string, string, string) {
|
||||
repository, _, _ := strings.Cut(l.ID, "@")
|
||||
name := repoName(repository)
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
}
|
||||
long := "too long"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
act := conditions.ResolverWords(o)
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
act = "It waits for you: release it, or stop it."
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
fmt.Sprintf("A delivery of %s has been %s %s, past its limit. %s", name, held, long, act),
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held)
|
||||
}
|
||||
|
||||
// causeWords is a hand-act's cause as a person says it.
|
||||
func causeWords(cause string) string {
|
||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
}
|
||||
|
||||
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
||||
func unitPlainWords(unit string) string {
|
||||
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
||||
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
||||
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
||||
if what == "" {
|
||||
return strings.ReplaceAll(name, ".", " ")
|
||||
}
|
||||
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(unit, ".", " ")
|
||||
}
|
||||
|
||||
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
||||
func resourcePlainWords(r inventory.ResourceHealth) string {
|
||||
if r.Kind == link.KindUnit {
|
||||
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
||||
}
|
||||
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return what + " keeps restarting"
|
||||
case "down":
|
||||
return what + " is not running"
|
||||
}
|
||||
if r.Check != "" {
|
||||
return what + " fails its " + r.Check + " check"
|
||||
}
|
||||
return what + " is not healthy"
|
||||
}
|
||||
|
||||
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
||||
// systemd's result, such as exit-code, or a sentence carrying it).
|
||||
func unitReasonWords(reason string) string {
|
||||
switch {
|
||||
case strings.Contains(reason, "start-limit"):
|
||||
return "failed too often to be started again"
|
||||
case strings.Contains(reason, "exit-code"):
|
||||
return "stopped with an error"
|
||||
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
||||
return "crashed"
|
||||
case strings.Contains(reason, "timeout"):
|
||||
return "timed out"
|
||||
}
|
||||
return "failed"
|
||||
}
|
||||
Reference in New Issue
Block a user