jschoubben
671e350f56
mesh/merge-gate pass: builds mesh-controller → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/438-a-check-that-never-answered-is-said delivered: every member is delivered
Review of #213: the queue is the controller's build queue, not the build seat's, and the merge check is the pair, so the headline says the pull request's merge check has not answered.
955 lines
45 KiB
Go
955 lines
45 KiB
Go
package main
|
|
|
|
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
|
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
|
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
|
//
|
|
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
|
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
|
// suite is held to having words, and to their being plain (plain_words_test.go).
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
type words = conditions.Words
|
|
|
|
// worded is a wording that needs only the condition's subject.
|
|
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
|
|
|
func init() {
|
|
for kind, fn := range plainWordings {
|
|
conditions.Wording(kind, fn)
|
|
}
|
|
}
|
|
|
|
// machineOr is the machine a condition concerns, or what to say when none is named.
|
|
func machineOr(o conditions.Observation, none string) string {
|
|
if o.Machine != "" {
|
|
return o.Machine
|
|
}
|
|
return none
|
|
}
|
|
|
|
// idPart is the n-th dotted part of a condition's id, or "".
|
|
func idPart(o conditions.Observation, n int) string {
|
|
parts := strings.Split(o.ID, ".")
|
|
if n < len(parts) {
|
|
return parts[n]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
|
func dataWords(o conditions.Observation) (what, module string) {
|
|
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
|
switch {
|
|
case module != "" && item != "":
|
|
return item + " of " + module, module
|
|
case module != "":
|
|
return "data of " + module, module
|
|
}
|
|
return "data", "a module"
|
|
}
|
|
|
|
var plainWordings = map[string]func(conditions.Observation) words{
|
|
// Machines and their node-engine.
|
|
"silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
w := words{Headline: m + " is not answering",
|
|
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
|
"normal.", m),
|
|
Resolved: m + " answers again"}
|
|
if o.Severity == conditions.Urgent {
|
|
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
|
|
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
|
|
"nothing changes anywhere until it is back.", m)
|
|
}
|
|
return w
|
|
}),
|
|
"sent-not-reported": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has not confirmed a change",
|
|
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
|
|
"It usually catches up by itself.", m, m),
|
|
Resolved: m + " applied the change"}
|
|
}),
|
|
"tools-silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Tools on " + m + " are not answering",
|
|
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
|
"the mesh until it does. A machine asleep is normal.", m, m),
|
|
Resolved: "Tools on " + m + " answer again"}
|
|
}),
|
|
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " waits for new instructions",
|
|
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
|
|
conditions.Capital(m)),
|
|
Resolved: m + " got its new instructions"}
|
|
}),
|
|
"declaration-refused": worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeMesh {
|
|
return words{Headline: "No machine can get new instructions",
|
|
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
|
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
|
"made. Machines keep running what they have.",
|
|
Resolved: "Machines can get new instructions again"}
|
|
}
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " cannot get new instructions",
|
|
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
|
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
|
|
"reaches it. It keeps running what it has.", m),
|
|
Resolved: m + " can get new instructions again"}
|
|
}),
|
|
kindAgentCanBecomeRoot: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Sessions on " + m + " could become root",
|
|
Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.",
|
|
Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+
|
|
"can take over the machine without you. The machine cannot show that this holds now, so an answer "+
|
|
"from your phone authorises nothing there until it does.", m),
|
|
Resolved: "Sessions on " + m + " cannot become root again"}
|
|
}),
|
|
"own-address-banned": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has banned the mesh",
|
|
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
|
|
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
|
"mesh is locked out of it.", m),
|
|
Resolved: m + " no longer bans the mesh"}
|
|
}),
|
|
"core-behind": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " runs an old node-engine",
|
|
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
|
|
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
|
|
Resolved: m + " runs the current node-engine"}
|
|
}),
|
|
kindMachineUnits: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Failed services on " + m,
|
|
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
|
|
Explanation: "No module manages them, so the mesh does not repair them.",
|
|
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
|
}),
|
|
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has a network problem",
|
|
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
|
"route. It often passes again once a network change settles.", conditions.Capital(m)),
|
|
Resolved: m + "'s network is fine again"}
|
|
}),
|
|
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + "'s network settings were changed",
|
|
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
|
"client. Mesh names may not resolve there until it is undone.", m),
|
|
Resolved: m + "'s network settings are the mesh's again"}
|
|
}),
|
|
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " cannot be reached",
|
|
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
|
"asleep is normal.", m),
|
|
Resolved: m + " can be reached again"}
|
|
}),
|
|
// A pending assignment that was not made (novox/hq issue 325, ADR 0261).
|
|
kindPendingEnded: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
module := idPart(o, 1)
|
|
if module == "" {
|
|
module = "a module"
|
|
}
|
|
return words{Headline: module + " was not put on " + m,
|
|
Needs: "decide whether to build it again or take the assignment back; the details say why.",
|
|
Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+
|
|
"register it, so it was not made.", module, m),
|
|
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
|
}),
|
|
kindBindingKept: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source is held on " + m,
|
|
Needs: "decide where its data lives; the details name both places.",
|
|
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
|
"kept the old one because switching would leave the data behind.", m),
|
|
Resolved: "Resolved: the data source on " + m + " is decided"}
|
|
}),
|
|
kindBindingMoving: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source moves on " + m,
|
|
Needs: "move its data to the new provider before the next push.",
|
|
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
}),
|
|
kindBindingMoved: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source moved on " + m,
|
|
Needs: "check that it found its data.",
|
|
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
}),
|
|
|
|
// Modules and providers.
|
|
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: conditions.Capital(thing) + " is not working",
|
|
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
|
|
Resolved: conditions.Capital(thing) + " works again"}
|
|
}),
|
|
kindReloginNeeded: worded(func(o conditions.Observation) words {
|
|
// A module's name may hold dots: it is the id without its machine.
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
return reloginWords(orModule(module), machineOr(o, "a machine"), false)
|
|
}),
|
|
kindUsedAsFound: worded(func(o conditions.Observation) words {
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
|
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
|
}),
|
|
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
|
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
node := machineOr(o, "a machine")
|
|
w := needsOperatorWords(orModule(module), node, nil)
|
|
return w
|
|
}),
|
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
|
if consumer == "" {
|
|
consumer = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
|
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
|
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
|
|
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
|
}),
|
|
kindProviderSilent: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: conditions.Capital(thing) + " went quiet",
|
|
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
|
"reporting.", conditions.Capital(thing)),
|
|
Resolved: conditions.Capital(thing) + " reports again"}
|
|
}),
|
|
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: "Retiring data on " + thing + " waits for you",
|
|
Needs: "approve or reject retiring it; the details list what would go.",
|
|
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
|
|
conditions.Capital(thing)),
|
|
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
|
}),
|
|
kindRetireRejected: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: "Unused data kept on " + thing,
|
|
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
|
|
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
|
}),
|
|
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
|
var where string
|
|
if o.Scope == conditions.ScopeProvider {
|
|
where = conditions.ThingWords(o)
|
|
} else {
|
|
what, _ := dataWords(o)
|
|
where = what + " on " + machineOr(o, "a machine")
|
|
}
|
|
return words{Headline: "Retired data waits for cleanup",
|
|
Needs: "delete it or keep it; nothing is deleted without your word.",
|
|
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
|
|
Resolved: "Resolved: the retired data is dealt with"}
|
|
}),
|
|
|
|
// Data.
|
|
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Data on " + m + " could not be measured",
|
|
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
|
"the data there. The next measurement may settle it.", m),
|
|
Resolved: "Data on " + m + " is measured again"}
|
|
}),
|
|
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " is unprotected",
|
|
Needs: "check the storage it is on.",
|
|
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
|
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is protected again"}
|
|
}),
|
|
kindDataMissing: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " is gone",
|
|
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
|
Needs: "restore it from a backup, or silence this " + FromMeshMCPServer,
|
|
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is back"}
|
|
}),
|
|
kindDataShrank: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
m := machineOr(o, "its machine")
|
|
if idPart(o, 1) == "dataset" {
|
|
what = "a dataset"
|
|
}
|
|
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
|
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
|
Needs: "restore the last good copy from a backup, or silence this " + FromMeshMCPServer,
|
|
Explanation: "More than half of what it held is gone within a week.",
|
|
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
|
}),
|
|
kindDataQuiet: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " stopped changing",
|
|
Needs: "check that what writes it still runs.",
|
|
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
|
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is written again"}
|
|
}),
|
|
kindBackupStale: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
m := machineOr(o, "its machine")
|
|
return words{Headline: "No recent backup of " + what,
|
|
Needs: fmt.Sprintf("check that backups run on %s.", m),
|
|
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
|
"changed since would be lost.", conditions.Capital(what), m),
|
|
Resolved: conditions.Capital(what) + " is backed up again"}
|
|
}),
|
|
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Storage on " + m + " is degraded",
|
|
Needs: fmt.Sprintf("check the disks of %s.", m),
|
|
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
|
|
Resolved: "Storage on " + m + " is whole again"}
|
|
}),
|
|
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
|
module := idPart(o, 1)
|
|
if o.Scope == conditions.ScopeProvider {
|
|
module = idPart(o, 2)
|
|
}
|
|
if module == "" {
|
|
module = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
|
Needs: "decide where its data lives: move the data, or move it back.",
|
|
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
|
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(module) + " has its data again"}
|
|
}),
|
|
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
|
consumer := idPart(o, 1)
|
|
if consumer == "" {
|
|
consumer = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
|
Needs: "decide which copy to keep.",
|
|
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
|
|
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
|
}),
|
|
|
|
// Deliveries and walks.
|
|
"stalled": worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeDelivery {
|
|
return words{Headline: "A delivery is held too long",
|
|
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
}
|
|
return words{Headline: "A delivery is stuck halfway",
|
|
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
}),
|
|
kindBatchNotCut: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Merged changes are not being delivered",
|
|
Explanation: "Merges collected for one delivery should have been planned and were not. Nothing is lost.",
|
|
Resolved: "Resolved: the merged changes are being delivered"}
|
|
}),
|
|
kindBatchBehindWalk: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Merged changes wait behind a slow delivery",
|
|
Explanation: "Merges collected for the next delivery wait for the delivery before them, which is taking " +
|
|
"longer than it should. Nothing is lost.",
|
|
Resolved: "Resolved: the merged changes no longer wait"}
|
|
}),
|
|
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A delivery is waiting to start",
|
|
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
|
"out) has not let it start yet.",
|
|
Resolved: "Resolved: the delivery is no longer waiting"}
|
|
}),
|
|
"merge-not-acted": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A merge was never picked up",
|
|
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
|
"sent. The mesh catches up on missed merges by itself.",
|
|
Resolved: "Resolved: the merge was picked up"}
|
|
}),
|
|
"ask-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A build has no result",
|
|
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
|
Resolved: "Resolved: the build has its result"}
|
|
}),
|
|
kindRolledBack: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
module := idPart(o, 0)
|
|
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
|
module = "a module"
|
|
}
|
|
if o.Scope == conditions.ScopeCore {
|
|
return words{Headline: "A core update was put back on " + m,
|
|
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
|
"before was put back and runs.", module, m),
|
|
Resolved: "Resolved: the core on " + m + " is healthy"}
|
|
}
|
|
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
|
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
|
"before, which runs. The change is not delivered until it is fixed.", module, m),
|
|
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
|
}),
|
|
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Putting back an update failed on " + m,
|
|
Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
|
|
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
|
|
Resolved: "Resolved: " + m + " runs a good build again"}
|
|
}),
|
|
"release-held": worded(func(o conditions.Observation) words {
|
|
return releaseHeldWords(nil, o.Also)
|
|
}),
|
|
"facts-stale": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Merge checks use outdated facts",
|
|
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
|
"by themselves.",
|
|
Resolved: "Merge checks use current facts again"}
|
|
}),
|
|
|
|
// The controller and the core.
|
|
"controller-deaf": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The controller stopped listening",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
|
"wait. Changes and repairs do not happen until it recovers.",
|
|
Resolved: "The controller listens again"}
|
|
}),
|
|
"self-check-silent": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh's self-check stopped",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
|
"Problems may go unnoticed until it runs again.",
|
|
Resolved: "The self-check runs again"}
|
|
}),
|
|
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh's watchdogs stopped",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
|
|
"missed signals are not noticed.",
|
|
Resolved: "The watchdogs run again"}
|
|
}),
|
|
"lease-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The controller lost its lease",
|
|
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
|
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
|
"not renew it, so two could act at once, or none.",
|
|
Resolved: "The controller holds its lease again"}
|
|
}),
|
|
"lease-split": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Two controllers may be acting",
|
|
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
|
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
|
|
Resolved: "One controller acts again"}
|
|
}),
|
|
"stale-writer": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Outdated instructions are being sent",
|
|
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
|
"were refused, which is harmless.",
|
|
Resolved: "Resolved: no more outdated instructions"}
|
|
}),
|
|
"status-slow": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh answers slowly",
|
|
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
|
Resolved: "The mesh answers quickly again"}
|
|
}),
|
|
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeBus {
|
|
return words{Headline: "The bus is not healthy",
|
|
Needs: "check the machine the bus runs on; the details say what fails.",
|
|
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
|
"and tool calls may fail until it recovers.",
|
|
Resolved: "The bus is healthy again"}
|
|
}
|
|
m := machineOr(o, "its machine")
|
|
return words{Headline: "The mesh's own software fails on " + m,
|
|
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
|
|
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
|
}),
|
|
"call-hung": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A tool call is hanging",
|
|
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
|
|
Resolved: "Resolved: the tool call ended"}
|
|
}),
|
|
"probe-failed": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A check of the mesh could not run",
|
|
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
|
|
"again every few minutes.",
|
|
Resolved: "The check runs again"}
|
|
}),
|
|
kindHealersBraked: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "All automatic repairs stopped",
|
|
Needs: "read the details and decide whether the repairs may run again.",
|
|
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
|
"looks like a loop, so all of them stopped.",
|
|
Resolved: "Automatic repairs run again"}
|
|
}),
|
|
"healer-wanted": worded(func(o conditions.Observation) words {
|
|
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
|
if o.Scope != conditions.ScopeMesh {
|
|
cause = "something"
|
|
}
|
|
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
|
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
|
|
"broken now.",
|
|
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
|
}),
|
|
|
|
// The bus and seats.
|
|
"resolver-wrong": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Name lookups fail on " + m,
|
|
Needs: fmt.Sprintf("check that %s is on and online.", m),
|
|
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
|
|
"machines that rely on it may not find each other.", m),
|
|
Resolved: "Name lookups work on " + m + " again"}
|
|
}),
|
|
"holder-silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
seat := idPart(o, 0)
|
|
if o.Scope != conditions.ScopeSeat {
|
|
seat = "a seat"
|
|
}
|
|
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
|
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
|
"machine asleep is normal.", seat, m),
|
|
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
|
}),
|
|
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus could not be fully set up",
|
|
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
|
|
"again with the next send.",
|
|
Resolved: "The bus is fully set up again"}
|
|
}),
|
|
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus is being upgraded",
|
|
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
|
|
Resolved: "The bus upgrade is done"}
|
|
}),
|
|
kindBusStepWaiting: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Sends wait for a bus upgrade",
|
|
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
|
Explanation: "A new version of the mesh's message system is built, and only a person installs it. Until " +
|
|
"then nothing else is sent to the machine that runs it.",
|
|
Resolved: "Resolved: the bus upgrade started, and sends go on"}
|
|
}),
|
|
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus upgrade failed",
|
|
Needs: "decide whether to put the bus back to the version before; the details say how.",
|
|
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
|
|
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
|
}),
|
|
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Messages pile up for a listener",
|
|
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
|
"healer restarts it if it stays.",
|
|
Resolved: "The listener caught up"}
|
|
}),
|
|
"consumer-wrong": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is set up wrong",
|
|
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
|
|
"it up again.",
|
|
Resolved: "The listener is set up right again"}
|
|
}),
|
|
"consumer-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is gone",
|
|
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
|
Resolved: "The listener is back"}
|
|
}),
|
|
"slow-consumer": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is too slow",
|
|
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
|
Resolved: "The listener keeps up again"}
|
|
}),
|
|
kindBusReconnects: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A client keeps losing the bus",
|
|
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
|
"While it reconnects, what it says and what it is asked waits.",
|
|
Resolved: "Resolved: the client stays connected"}
|
|
}),
|
|
"max-deliveries": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener gave up on messages",
|
|
Needs: "deliver them again or drop them, from the mesh MCP server.",
|
|
Explanation: "A listener on the bus could not handle messages after several tries, so what they asked " +
|
|
"for was not done. The mesh keeps them until you deliver them again or drop them.",
|
|
Resolved: "Resolved: the messages given up on were delivered again or dropped"}
|
|
}),
|
|
"refused": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus refuses some messages",
|
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
|
Resolved: "Resolved: the bus takes the messages again"}
|
|
}),
|
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Part of the bus's storage is wrong",
|
|
Needs: "check the machine the bus runs on; the details say what is missing.",
|
|
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
|
Resolved: "The bus's storage is right again"}
|
|
}),
|
|
"archives-unheld": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Kept builds are about to be cleaned up",
|
|
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
|
|
Resolved: "Resolved: every kept build is claimed"}
|
|
}),
|
|
"archives-missing": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Kept builds are missing",
|
|
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
|
|
Resolved: "Resolved: the kept builds are back"}
|
|
}),
|
|
}
|
|
|
|
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
|
func humanDuration(d time.Duration) string {
|
|
switch {
|
|
case d < 2*time.Minute:
|
|
return "a minute"
|
|
case d < 2*time.Hour:
|
|
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%d hours", int(d.Hours()))
|
|
}
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
}
|
|
|
|
// repoName is a repository as a person says it: its name without its owner.
|
|
func repoName(repository string) string {
|
|
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
|
return repository[i+1:]
|
|
}
|
|
return repository
|
|
}
|
|
|
|
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
|
func namesWords(names []string, most int) string {
|
|
switch {
|
|
case len(names) == 0:
|
|
return ""
|
|
case len(names) == 1:
|
|
return names[0]
|
|
case len(names) <= most:
|
|
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
|
}
|
|
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
|
}
|
|
|
|
// planModules are the modules a walk moves, sorted.
|
|
func planModules(p inventory.Plan) []string {
|
|
out := make([]string, 0, len(p.Modules))
|
|
for m := range p.Modules {
|
|
out = append(out, m)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
|
// their count and the repository.
|
|
func deliveryWhat(modules []string, repository string) string {
|
|
if len(modules) > 0 && len(modules) <= 2 {
|
|
return namesWords(modules, 2)
|
|
}
|
|
if len(modules) > 2 {
|
|
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
|
}
|
|
return repoName(repository)
|
|
}
|
|
|
|
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
|
func deliveryName(modules []string, repository string) string {
|
|
if len(modules) > 0 && len(modules) <= 2 {
|
|
return namesWords(modules, 2) + " delivery"
|
|
}
|
|
return repoName(repository) + " delivery"
|
|
}
|
|
|
|
// walkWaitingWords explains a walk waiting for its delivery's word.
|
|
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
|
|
what := deliveryWhat(w.modules, w.repository)
|
|
if severity == conditions.Urgent {
|
|
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
|
|
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
|
|
}
|
|
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
|
|
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
|
|
what, humanDuration(in), humanDuration(waitUrgentAfter))
|
|
}
|
|
|
|
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
|
// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying
|
|
// where they are given without a channel, and the ask's text drops that (askText).
|
|
func waitingNeeds(severity conditions.Severity) string {
|
|
if severity == conditions.Urgent {
|
|
return "start it, or stop it, " + FromMeshMCPServer
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's
|
|
// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound.
|
|
func waitingActions(plan string, severity conditions.Severity) []conditions.Action {
|
|
if severity != conditions.Urgent || plan == "" {
|
|
return nil
|
|
}
|
|
return []conditions.Action{
|
|
{Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
|
|
{Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
|
|
}
|
|
}
|
|
|
|
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
|
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
|
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
|
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
|
unit := ""
|
|
for _, r := range rs {
|
|
if strings.Contains(r.Reason, "relogin needed") {
|
|
return reloginNeeds(node)
|
|
}
|
|
if r.Kind == link.KindUnit && unit == "" {
|
|
unit = unitPlainWords(r.Target)
|
|
}
|
|
}
|
|
if unit != "" {
|
|
// Also asked of the operator (moduleActions); the ask's text drops where (askText).
|
|
return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// moduleActions are the answers to a module unhealthy on a machine: restart its failed service there,
|
|
// approved by the operator (novox/hq ADR 0259) — none when the mesh restarts it, or a new login is what it
|
|
// waits for.
|
|
func moduleActions(node string, rs []inventory.ResourceHealth) []conditions.Action {
|
|
for _, r := range rs {
|
|
if strings.Contains(r.Reason, "relogin needed") {
|
|
return nil
|
|
}
|
|
}
|
|
for _, r := range rs {
|
|
if r.Kind != link.KindUnit || r.Target == "" {
|
|
continue
|
|
}
|
|
scope := "system"
|
|
if r.Account != "" {
|
|
scope = "user"
|
|
}
|
|
return []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: node,
|
|
Level: conditions.LevelApprove, Arguments: map[string]string{"unit": r.Target, "scope": scope}}}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP
|
|
// server (the glossary's word; "console" is retired): the answer is not an
|
|
// acknowledgement, so it is given where the operator is known to be the one asking, until answers are
|
|
// authorised (to-be 46 phases 5 and 6).
|
|
const FromMeshMCPServer = "from the mesh MCP server; this notification cannot do it."
|
|
|
|
// releaseHeldWords are the plain words of updates held after a walk failed: which modules wait,
|
|
// on which machines, and where the operator releases them (ADR 0258: a release is not an acknowledgement, so
|
|
// no notification gives it). Raised with the modules and machines (backlogObservation); the kind's fallback
|
|
// knows neither.
|
|
func releaseHeldWords(modules, machines []string) words {
|
|
what := "Some module updates"
|
|
headline := "Updates wait for your release"
|
|
if len(modules) > 0 {
|
|
what = "Updates of " + namesWords(modules, 3)
|
|
if h := what + " wait for your release"; len(h) <= conditions.HeadlineMax {
|
|
headline = h
|
|
} else if h := fmt.Sprintf("%d module updates wait for your release", len(modules)); len(h) <= conditions.HeadlineMax {
|
|
headline = h
|
|
}
|
|
}
|
|
where := ""
|
|
if len(machines) > 0 {
|
|
where = " on " + namesWords(machines, 4)
|
|
}
|
|
return words{Headline: headline,
|
|
Needs: "release them " + FromMeshMCPServer,
|
|
Explanation: fmt.Sprintf("%s%s wait for a person to release them, because the last walk failed."+
|
|
" They are not delivered until then, and stay held if you leave them.", what, where),
|
|
Resolved: "Resolved: the held updates are released"}
|
|
}
|
|
|
|
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
|
func reloginNeeds(node string) string {
|
|
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
|
}
|
|
|
|
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
|
func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
|
actions []conditions.Action) {
|
|
repository, _, _ := strings.Cut(l.ID, "@")
|
|
name := repoName(repository)
|
|
if l.State == "unannounced" {
|
|
// A pull request the forge never announced (novox/hq issue 347): no delivery exists, so there is nothing to
|
|
// stop, release or close; a new commit on its branch is announced and checked.
|
|
long := "for too long"
|
|
if d, err := time.ParseDuration(l.For); err == nil {
|
|
long = "for " + humanDuration(d)
|
|
}
|
|
if o.Resolver == conditions.ResolverOperator {
|
|
needs = "push a new commit to its branch; the forge announces it and the mesh checks it."
|
|
}
|
|
pull := "A pull request of " + name
|
|
if l.Number > 0 {
|
|
pull = fmt.Sprintf("Pull request %s #%d", name, l.Number)
|
|
}
|
|
return fmt.Sprintf("%s has had no merge check %s", pull, long),
|
|
fmt.Sprintf("%s has been open %s on a branch that requires the merge check, and the mesh was never asked "+
|
|
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
|
|
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
|
|
}
|
|
if l.State == "unanswered" {
|
|
return unansweredWords(l, o)
|
|
}
|
|
held := l.State
|
|
if held == "" {
|
|
held = "held"
|
|
}
|
|
long := "too long"
|
|
if d, err := time.ParseDuration(l.For); err == nil {
|
|
long = "for " + humanDuration(d)
|
|
}
|
|
if o.Resolver == conditions.ResolverOperator {
|
|
// Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the
|
|
// router says where each can be answered, so the words do not.
|
|
release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"id": l.ID, "why": ""}}
|
|
stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"id": l.ID, "why": ""}}
|
|
switch held {
|
|
case "held":
|
|
needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop}
|
|
case "ready", "checked":
|
|
needs = "merge its pull request, or close it."
|
|
default:
|
|
needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop}
|
|
}
|
|
}
|
|
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
|
fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
|
|
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
|
}
|
|
|
|
// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered
|
|
// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
|
|
// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
|
|
// at the build queue, where a check that never ran may still wait.
|
|
func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
|
actions []conditions.Action) {
|
|
repository, _, _ := strings.Cut(l.ID, "@")
|
|
pull := "A pull request of " + repoName(repository)
|
|
if l.Number > 0 {
|
|
pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number)
|
|
}
|
|
long, limit := "for too long", ""
|
|
if d, err := time.ParseDuration(l.For); err == nil {
|
|
long = "for " + humanDuration(d)
|
|
}
|
|
if d, err := time.ParseDuration(l.Bound); err == nil {
|
|
limit = ", past its limit of " + humanDuration(d)
|
|
}
|
|
which := "Its merge check"
|
|
if len(l.Waiting) > 0 {
|
|
which = "Its merge check (" + strings.Join(l.Waiting, ", ") + ")"
|
|
}
|
|
if o.Resolver == conditions.ResolverOperator {
|
|
needs = "push a new commit to its branch, which asks the check again, or see whether its check still waits " +
|
|
"in the build queue: mesh-controller.queue."
|
|
}
|
|
return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
|
|
fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+
|
|
"It cannot merge until its check answers.", pull, which, long, limit),
|
|
fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil
|
|
}
|
|
|
|
// causeWords is a hand-act's cause as a person says it.
|
|
func causeWords(cause string) string {
|
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
|
}
|
|
|
|
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
|
func unitPlainWords(unit string) string {
|
|
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
|
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
|
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
|
if what == "" {
|
|
return strings.ReplaceAll(name, ".", " ")
|
|
}
|
|
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
|
}
|
|
}
|
|
return strings.ReplaceAll(unit, ".", " ")
|
|
}
|
|
|
|
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
|
func resourcePlainWords(r inventory.ResourceHealth) string {
|
|
if r.Kind == link.KindUnit {
|
|
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
|
}
|
|
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return what + " keeps restarting"
|
|
case "down":
|
|
return what + " is not running"
|
|
}
|
|
if r.Check != "" {
|
|
return what + " fails its " + r.Check + " check"
|
|
}
|
|
return what + " is not healthy"
|
|
}
|
|
|
|
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
|
// systemd's result, such as exit-code, or a sentence carrying it).
|
|
func unitReasonWords(reason string) string {
|
|
switch {
|
|
case strings.Contains(reason, "start-limit"):
|
|
return "failed too often to be started again"
|
|
case strings.Contains(reason, "exit-code"):
|
|
return "stopped with an error"
|
|
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
|
return "crashed"
|
|
case strings.Contains(reason, "timeout"):
|
|
return "timed out"
|
|
}
|
|
return "failed"
|
|
}
|
|
|
|
// orModule is a module's name, or "a module" when none is known.
|
|
func orModule(m string) string {
|
|
if m == "" {
|
|
return "a module"
|
|
}
|
|
return m
|
|
}
|