jochen
1e04670052
Each verb ran as a process that dialled the bus, so hundreds of short connections an hour, all named mesh-controller, hid any client reconnecting in a loop (hq issue 327). D15 now says a user whose connections keep dropping.
812 lines
38 KiB
Go
812 lines
38 KiB
Go
package main
|
|
|
|
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
|
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
|
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
|
//
|
|
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
|
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
|
// suite is held to having words, and to their being plain (plain_words_test.go).
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
type words = conditions.Words
|
|
|
|
// worded is a wording that needs only the condition's subject.
|
|
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
|
|
|
func init() {
|
|
for kind, fn := range plainWordings {
|
|
conditions.Wording(kind, fn)
|
|
}
|
|
}
|
|
|
|
// machineOr is the machine a condition concerns, or what to say when none is named.
|
|
func machineOr(o conditions.Observation, none string) string {
|
|
if o.Machine != "" {
|
|
return o.Machine
|
|
}
|
|
return none
|
|
}
|
|
|
|
// idPart is the n-th dotted part of a condition's id, or "".
|
|
func idPart(o conditions.Observation, n int) string {
|
|
parts := strings.Split(o.ID, ".")
|
|
if n < len(parts) {
|
|
return parts[n]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
|
func dataWords(o conditions.Observation) (what, module string) {
|
|
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
|
switch {
|
|
case module != "" && item != "":
|
|
return item + " of " + module, module
|
|
case module != "":
|
|
return "data of " + module, module
|
|
}
|
|
return "data", "a module"
|
|
}
|
|
|
|
var plainWordings = map[string]func(conditions.Observation) words{
|
|
// Machines and their node-engine.
|
|
"silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
w := words{Headline: m + " is not answering",
|
|
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
|
"normal.", m),
|
|
Resolved: m + " answers again"}
|
|
if o.Severity == conditions.Urgent {
|
|
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
|
|
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
|
|
"nothing changes anywhere until it is back.", m)
|
|
}
|
|
return w
|
|
}),
|
|
"sent-not-reported": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has not confirmed a change",
|
|
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
|
|
"It usually catches up by itself.", m, m),
|
|
Resolved: m + " applied the change"}
|
|
}),
|
|
"tools-silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Tools on " + m + " are not answering",
|
|
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
|
"the mesh until it does. A machine asleep is normal.", m, m),
|
|
Resolved: "Tools on " + m + " answer again"}
|
|
}),
|
|
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " waits for new instructions",
|
|
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
|
|
conditions.Capital(m)),
|
|
Resolved: m + " got its new instructions"}
|
|
}),
|
|
"declaration-refused": worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeMesh {
|
|
return words{Headline: "No machine can get new instructions",
|
|
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
|
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
|
"made. Machines keep running what they have.",
|
|
Resolved: "Machines can get new instructions again"}
|
|
}
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " cannot get new instructions",
|
|
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
|
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
|
|
"reaches it. It keeps running what it has.", m),
|
|
Resolved: m + " can get new instructions again"}
|
|
}),
|
|
"own-address-banned": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has banned the mesh",
|
|
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
|
|
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
|
"mesh is locked out of it.", m),
|
|
Resolved: m + " no longer bans the mesh"}
|
|
}),
|
|
"core-behind": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " runs an old node-engine",
|
|
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
|
|
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
|
|
Resolved: m + " runs the current node-engine"}
|
|
}),
|
|
kindMachineUnits: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Failed services on " + m,
|
|
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
|
|
Explanation: "No module manages them, so the mesh does not repair them.",
|
|
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
|
}),
|
|
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has a network problem",
|
|
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
|
"route. It often passes again once a network change settles.", conditions.Capital(m)),
|
|
Resolved: m + "'s network is fine again"}
|
|
}),
|
|
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + "'s network settings were changed",
|
|
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
|
"client. Mesh names may not resolve there until it is undone.", m),
|
|
Resolved: m + "'s network settings are the mesh's again"}
|
|
}),
|
|
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " cannot be reached",
|
|
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
|
"asleep is normal.", m),
|
|
Resolved: m + " can be reached again"}
|
|
}),
|
|
// A pending assignment that was not made (novox/hq issue 325, ADR 0261).
|
|
kindPendingEnded: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
module := idPart(o, 1)
|
|
if module == "" {
|
|
module = "a module"
|
|
}
|
|
return words{Headline: module + " was not put on " + m,
|
|
Needs: "decide whether to build it again or take the assignment back; the details say why.",
|
|
Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+
|
|
"register it, so it was not made.", module, m),
|
|
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
|
}),
|
|
kindBindingKept: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source is held on " + m,
|
|
Needs: "decide where its data lives; the details name both places.",
|
|
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
|
"kept the old one because switching would leave the data behind.", m),
|
|
Resolved: "Resolved: the data source on " + m + " is decided"}
|
|
}),
|
|
kindBindingMoving: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source moves on " + m,
|
|
Needs: "move its data to the new provider before the next push.",
|
|
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
}),
|
|
kindBindingMoved: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source moved on " + m,
|
|
Needs: "check that it found its data.",
|
|
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
}),
|
|
|
|
// Modules and providers.
|
|
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: conditions.Capital(thing) + " is not working",
|
|
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
|
|
Resolved: conditions.Capital(thing) + " works again"}
|
|
}),
|
|
kindReloginNeeded: worded(func(o conditions.Observation) words {
|
|
// A module's name may hold dots: it is the id without its machine.
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
return reloginWords(orModule(module), machineOr(o, "a machine"), false)
|
|
}),
|
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
|
if consumer == "" {
|
|
consumer = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
|
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
|
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
|
|
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
|
}),
|
|
kindProviderSilent: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: conditions.Capital(thing) + " went quiet",
|
|
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
|
"reporting.", conditions.Capital(thing)),
|
|
Resolved: conditions.Capital(thing) + " reports again"}
|
|
}),
|
|
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: "Retiring data on " + thing + " waits for you",
|
|
Needs: "approve or reject retiring it; the details list what would go.",
|
|
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
|
|
conditions.Capital(thing)),
|
|
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
|
}),
|
|
kindRetireRejected: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: "Unused data kept on " + thing,
|
|
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
|
|
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
|
}),
|
|
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
|
var where string
|
|
if o.Scope == conditions.ScopeProvider {
|
|
where = conditions.ThingWords(o)
|
|
} else {
|
|
what, _ := dataWords(o)
|
|
where = what + " on " + machineOr(o, "a machine")
|
|
}
|
|
return words{Headline: "Retired data waits for cleanup",
|
|
Needs: "delete it or keep it; nothing is deleted without your word.",
|
|
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
|
|
Resolved: "Resolved: the retired data is dealt with"}
|
|
}),
|
|
|
|
// Data.
|
|
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Data on " + m + " could not be measured",
|
|
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
|
"the data there. The next measurement may settle it.", m),
|
|
Resolved: "Data on " + m + " is measured again"}
|
|
}),
|
|
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " is unprotected",
|
|
Needs: "check the storage it is on.",
|
|
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
|
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is protected again"}
|
|
}),
|
|
kindDataMissing: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " is gone",
|
|
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
|
Needs: "restore it from a backup, or silence this " + FromMeshMCPServer,
|
|
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is back"}
|
|
}),
|
|
kindDataShrank: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
m := machineOr(o, "its machine")
|
|
if idPart(o, 1) == "dataset" {
|
|
what = "a dataset"
|
|
}
|
|
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
|
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
|
Needs: "restore the last good copy from a backup, or silence this " + FromMeshMCPServer,
|
|
Explanation: "More than half of what it held is gone within a week.",
|
|
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
|
}),
|
|
kindDataQuiet: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " stopped changing",
|
|
Needs: "check that what writes it still runs.",
|
|
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
|
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is written again"}
|
|
}),
|
|
kindBackupStale: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
m := machineOr(o, "its machine")
|
|
return words{Headline: "No recent backup of " + what,
|
|
Needs: fmt.Sprintf("check that backups run on %s.", m),
|
|
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
|
"changed since would be lost.", conditions.Capital(what), m),
|
|
Resolved: conditions.Capital(what) + " is backed up again"}
|
|
}),
|
|
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Storage on " + m + " is degraded",
|
|
Needs: fmt.Sprintf("check the disks of %s.", m),
|
|
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
|
|
Resolved: "Storage on " + m + " is whole again"}
|
|
}),
|
|
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
|
module := idPart(o, 1)
|
|
if o.Scope == conditions.ScopeProvider {
|
|
module = idPart(o, 2)
|
|
}
|
|
if module == "" {
|
|
module = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
|
Needs: "decide where its data lives: move the data, or move it back.",
|
|
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
|
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(module) + " has its data again"}
|
|
}),
|
|
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
|
consumer := idPart(o, 1)
|
|
if consumer == "" {
|
|
consumer = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
|
Needs: "decide which copy to keep.",
|
|
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
|
|
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
|
}),
|
|
|
|
// Deliveries and walks.
|
|
"stalled": worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeDelivery {
|
|
return words{Headline: "A delivery is held too long",
|
|
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
}
|
|
return words{Headline: "A delivery is stuck halfway",
|
|
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
}),
|
|
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A delivery is waiting to start",
|
|
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
|
"out) has not let it start yet.",
|
|
Resolved: "Resolved: the delivery is no longer waiting"}
|
|
}),
|
|
"merge-not-acted": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A merge was never picked up",
|
|
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
|
"sent. The mesh catches up on missed merges by itself.",
|
|
Resolved: "Resolved: the merge was picked up"}
|
|
}),
|
|
"ask-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A build has no result",
|
|
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
|
Resolved: "Resolved: the build has its result"}
|
|
}),
|
|
kindRolledBack: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
module := idPart(o, 0)
|
|
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
|
module = "a module"
|
|
}
|
|
if o.Scope == conditions.ScopeCore {
|
|
return words{Headline: "A core update was put back on " + m,
|
|
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
|
"before was put back and runs.", module, m),
|
|
Resolved: "Resolved: the core on " + m + " is healthy"}
|
|
}
|
|
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
|
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
|
"before, which runs. The change is not delivered until it is fixed.", module, m),
|
|
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
|
}),
|
|
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Putting back an update failed on " + m,
|
|
Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
|
|
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
|
|
Resolved: "Resolved: " + m + " runs a good build again"}
|
|
}),
|
|
"release-held": worded(func(o conditions.Observation) words {
|
|
return releaseHeldWords(nil, o.Also)
|
|
}),
|
|
"facts-stale": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Merge checks use outdated facts",
|
|
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
|
"by themselves.",
|
|
Resolved: "Merge checks use current facts again"}
|
|
}),
|
|
|
|
// The controller and the core.
|
|
"controller-deaf": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The controller stopped listening",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
|
"wait. Changes and repairs do not happen until it recovers.",
|
|
Resolved: "The controller listens again"}
|
|
}),
|
|
"self-check-silent": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh's self-check stopped",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
|
"Problems may go unnoticed until it runs again.",
|
|
Resolved: "The self-check runs again"}
|
|
}),
|
|
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh's watchdogs stopped",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
|
|
"missed signals are not noticed.",
|
|
Resolved: "The watchdogs run again"}
|
|
}),
|
|
"lease-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The controller lost its lease",
|
|
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
|
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
|
"not renew it, so two could act at once, or none.",
|
|
Resolved: "The controller holds its lease again"}
|
|
}),
|
|
"lease-split": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Two controllers may be acting",
|
|
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
|
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
|
|
Resolved: "One controller acts again"}
|
|
}),
|
|
"stale-writer": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Outdated instructions are being sent",
|
|
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
|
"were refused, which is harmless.",
|
|
Resolved: "Resolved: no more outdated instructions"}
|
|
}),
|
|
"status-slow": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh answers slowly",
|
|
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
|
Resolved: "The mesh answers quickly again"}
|
|
}),
|
|
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeBus {
|
|
return words{Headline: "The bus is not healthy",
|
|
Needs: "check the machine the bus runs on; the details say what fails.",
|
|
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
|
"and tool calls may fail until it recovers.",
|
|
Resolved: "The bus is healthy again"}
|
|
}
|
|
m := machineOr(o, "its machine")
|
|
return words{Headline: "The mesh's own software fails on " + m,
|
|
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
|
|
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
|
}),
|
|
"call-hung": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A tool call is hanging",
|
|
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
|
|
Resolved: "Resolved: the tool call ended"}
|
|
}),
|
|
"probe-failed": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A check of the mesh could not run",
|
|
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
|
|
"again every few minutes.",
|
|
Resolved: "The check runs again"}
|
|
}),
|
|
kindHealersBraked: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "All automatic repairs stopped",
|
|
Needs: "read the details and decide whether the repairs may run again.",
|
|
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
|
"looks like a loop, so all of them stopped.",
|
|
Resolved: "Automatic repairs run again"}
|
|
}),
|
|
"healer-wanted": worded(func(o conditions.Observation) words {
|
|
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
|
if o.Scope != conditions.ScopeMesh {
|
|
cause = "something"
|
|
}
|
|
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
|
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
|
|
"broken now.",
|
|
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
|
}),
|
|
|
|
// The bus and seats.
|
|
"resolver-wrong": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Name lookups fail on " + m,
|
|
Needs: fmt.Sprintf("check that %s is on and online.", m),
|
|
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
|
|
"machines that rely on it may not find each other.", m),
|
|
Resolved: "Name lookups work on " + m + " again"}
|
|
}),
|
|
"holder-silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
seat := idPart(o, 0)
|
|
if o.Scope != conditions.ScopeSeat {
|
|
seat = "a seat"
|
|
}
|
|
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
|
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
|
"machine asleep is normal.", seat, m),
|
|
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
|
}),
|
|
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus could not be fully set up",
|
|
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
|
|
"again with the next send.",
|
|
Resolved: "The bus is fully set up again"}
|
|
}),
|
|
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus is being upgraded",
|
|
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
|
|
Resolved: "The bus upgrade is done"}
|
|
}),
|
|
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus upgrade failed",
|
|
Needs: "decide whether to put the bus back to the version before; the details say how.",
|
|
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
|
|
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
|
}),
|
|
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Messages pile up for a listener",
|
|
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
|
"healer restarts it if it stays.",
|
|
Resolved: "The listener caught up"}
|
|
}),
|
|
"consumer-wrong": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is set up wrong",
|
|
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
|
|
"it up again.",
|
|
Resolved: "The listener is set up right again"}
|
|
}),
|
|
"consumer-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is gone",
|
|
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
|
Resolved: "The listener is back"}
|
|
}),
|
|
"slow-consumer": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is too slow",
|
|
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
|
Resolved: "The listener keeps up again"}
|
|
}),
|
|
kindBusReconnects: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A client keeps losing the bus",
|
|
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
|
"While it reconnects, what it says and what it is asked waits.",
|
|
Resolved: "Resolved: the client stays connected"}
|
|
}),
|
|
"max-deliveries": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener gave up on messages",
|
|
Needs: "deliver them again or drop them, from the mesh MCP server.",
|
|
Explanation: "A listener on the bus could not handle messages after several tries, so what they asked " +
|
|
"for was not done. The mesh keeps them until you deliver them again or drop them.",
|
|
Resolved: "Resolved: the messages given up on were delivered again or dropped"}
|
|
}),
|
|
"refused": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus refuses some messages",
|
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
|
Resolved: "Resolved: the bus takes the messages again"}
|
|
}),
|
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Part of the bus's storage is wrong",
|
|
Needs: "check the machine the bus runs on; the details say what is missing.",
|
|
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
|
Resolved: "The bus's storage is right again"}
|
|
}),
|
|
"archives-unheld": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Kept builds are about to be cleaned up",
|
|
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
|
|
Resolved: "Resolved: every kept build is claimed"}
|
|
}),
|
|
"archives-missing": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Kept builds are missing",
|
|
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
|
|
Resolved: "Resolved: the kept builds are back"}
|
|
}),
|
|
}
|
|
|
|
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
|
func humanDuration(d time.Duration) string {
|
|
switch {
|
|
case d < 2*time.Minute:
|
|
return "a minute"
|
|
case d < 2*time.Hour:
|
|
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%d hours", int(d.Hours()))
|
|
}
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
}
|
|
|
|
// repoName is a repository as a person says it: its name without its owner.
|
|
func repoName(repository string) string {
|
|
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
|
return repository[i+1:]
|
|
}
|
|
return repository
|
|
}
|
|
|
|
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
|
func namesWords(names []string, most int) string {
|
|
switch {
|
|
case len(names) == 0:
|
|
return ""
|
|
case len(names) == 1:
|
|
return names[0]
|
|
case len(names) <= most:
|
|
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
|
}
|
|
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
|
}
|
|
|
|
// planModules are the modules a walk moves, sorted.
|
|
func planModules(p inventory.Plan) []string {
|
|
out := make([]string, 0, len(p.Modules))
|
|
for m := range p.Modules {
|
|
out = append(out, m)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
|
// their count and the repository.
|
|
func deliveryWhat(modules []string, repository string) string {
|
|
if len(modules) > 0 && len(modules) <= 2 {
|
|
return namesWords(modules, 2)
|
|
}
|
|
if len(modules) > 2 {
|
|
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
|
}
|
|
return repoName(repository)
|
|
}
|
|
|
|
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
|
func deliveryName(modules []string, repository string) string {
|
|
if len(modules) > 0 && len(modules) <= 2 {
|
|
return namesWords(modules, 2) + " delivery"
|
|
}
|
|
return repoName(repository) + " delivery"
|
|
}
|
|
|
|
// walkWaitingWords explains a walk waiting for its delivery's word.
|
|
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
|
|
what := deliveryWhat(w.modules, w.repository)
|
|
if severity == conditions.Urgent {
|
|
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
|
|
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
|
|
}
|
|
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
|
|
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
|
|
what, humanDuration(in), humanDuration(waitUrgentAfter))
|
|
}
|
|
|
|
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
|
func waitingNeeds(severity conditions.Severity) string {
|
|
if severity == conditions.Urgent {
|
|
return "start it, or stop it, " + FromMeshMCPServer
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
|
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
|
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
|
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
|
unit := ""
|
|
for _, r := range rs {
|
|
if strings.Contains(r.Reason, "relogin needed") {
|
|
return reloginNeeds(node)
|
|
}
|
|
if r.Kind == link.KindUnit && unit == "" {
|
|
unit = unitPlainWords(r.Target)
|
|
}
|
|
}
|
|
if unit != "" {
|
|
return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP
|
|
// server (the glossary's word; "console" is retired): the answer is not an
|
|
// acknowledgement, so it is given where the operator is known to be the one asking, until answers are
|
|
// authorised (to-be 46 phases 5 and 6).
|
|
const FromMeshMCPServer = "from the mesh MCP server; this notification cannot do it."
|
|
|
|
// releaseHeldWords are the plain words of updates held after a walk failed: which modules wait,
|
|
// on which machines, and where the operator releases them (ADR 0258: a release is not an acknowledgement, so
|
|
// no notification gives it). Raised with the modules and machines (backlogObservation); the kind's fallback
|
|
// knows neither.
|
|
func releaseHeldWords(modules, machines []string) words {
|
|
what := "Some module updates"
|
|
headline := "Updates wait for your release"
|
|
if len(modules) > 0 {
|
|
what = "Updates of " + namesWords(modules, 3)
|
|
if h := what + " wait for your release"; len(h) <= conditions.HeadlineMax {
|
|
headline = h
|
|
} else if h := fmt.Sprintf("%d module updates wait for your release", len(modules)); len(h) <= conditions.HeadlineMax {
|
|
headline = h
|
|
}
|
|
}
|
|
where := ""
|
|
if len(machines) > 0 {
|
|
where = " on " + namesWords(machines, 4)
|
|
}
|
|
return words{Headline: headline,
|
|
Needs: "release them " + FromMeshMCPServer,
|
|
Explanation: fmt.Sprintf("%s%s wait for a person to release them, because the last walk failed."+
|
|
" They are not delivered until then, and stay held if you leave them.", what, where),
|
|
Resolved: "Resolved: the held updates are released"}
|
|
}
|
|
|
|
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
|
func reloginNeeds(node string) string {
|
|
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
|
}
|
|
|
|
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
|
func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
|
actions []conditions.Action) {
|
|
repository, _, _ := strings.Cut(l.ID, "@")
|
|
name := repoName(repository)
|
|
held := l.State
|
|
if held == "" {
|
|
held = "held"
|
|
}
|
|
long := "too long"
|
|
if d, err := time.ParseDuration(l.For); err == nil {
|
|
long = "for " + humanDuration(d)
|
|
}
|
|
if o.Resolver == conditions.ResolverOperator {
|
|
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
|
|
// performs it (ADR 0258).
|
|
switch held {
|
|
case "held":
|
|
needs = "release it, or stop it, " + FromMeshMCPServer
|
|
case "ready", "checked":
|
|
needs = "merge its pull request, or close it."
|
|
default:
|
|
needs = "stop it " + FromMeshMCPServer
|
|
}
|
|
}
|
|
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
|
fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
|
|
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
|
}
|
|
|
|
// causeWords is a hand-act's cause as a person says it.
|
|
func causeWords(cause string) string {
|
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
|
}
|
|
|
|
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
|
func unitPlainWords(unit string) string {
|
|
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
|
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
|
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
|
if what == "" {
|
|
return strings.ReplaceAll(name, ".", " ")
|
|
}
|
|
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
|
}
|
|
}
|
|
return strings.ReplaceAll(unit, ".", " ")
|
|
}
|
|
|
|
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
|
func resourcePlainWords(r inventory.ResourceHealth) string {
|
|
if r.Kind == link.KindUnit {
|
|
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
|
}
|
|
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return what + " keeps restarting"
|
|
case "down":
|
|
return what + " is not running"
|
|
}
|
|
if r.Check != "" {
|
|
return what + " fails its " + r.Check + " check"
|
|
}
|
|
return what + " is not healthy"
|
|
}
|
|
|
|
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
|
// systemd's result, such as exit-code, or a sentence carrying it).
|
|
func unitReasonWords(reason string) string {
|
|
switch {
|
|
case strings.Contains(reason, "start-limit"):
|
|
return "failed too often to be started again"
|
|
case strings.Contains(reason, "exit-code"):
|
|
return "stopped with an error"
|
|
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
|
return "crashed"
|
|
case strings.Contains(reason, "timeout"):
|
|
return "timed out"
|
|
}
|
|
return "failed"
|
|
}
|
|
|
|
// orModule is a module's name, or "a module" when none is known.
|
|
func orModule(m string) string {
|
|
if m == "" {
|
|
return "a module"
|
|
}
|
|
return m
|
|
}
|