jschoubben
e40b1e70db
The start composed every kept composition of every module on every node, each a whole node composed in memory under the node's lock: 10 to 50 minutes at a start, holding take-ins back. Now the writes of mesh-config_resolved are read from the durable consumer mesh-controller-config-follow, acknowledged once the change is composed, so a restart composes again only what was written since the last change handled, and a first start nothing it did not hear. The node's lock is taken for each write alone. A consumer that cannot be made or read raises config-not-followed and is tried again a minute later, cleared once it reads. The controller's account is granted that consumer's acknowledgements where mesh-config serves, as its verb is. Research 072: milestone 2, the composer reacting to the change of an input.
854 lines
42 KiB
Go
854 lines
42 KiB
Go
package main
|
|
|
|
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
|
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
|
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
|
//
|
|
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
|
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
|
// suite is held to having words, and to their being plain (plain_words_test.go).
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
type words = conditions.Words
|
|
|
|
// worded is a wording that needs only the condition's subject.
|
|
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
|
|
|
func init() {
|
|
for kind, fn := range plainWordings {
|
|
conditions.Wording(kind, fn)
|
|
}
|
|
}
|
|
|
|
// machineOr is the machine a condition concerns, or what to say when none is named.
|
|
func machineOr(o conditions.Observation, none string) string {
|
|
if o.Machine != "" {
|
|
return o.Machine
|
|
}
|
|
return none
|
|
}
|
|
|
|
// idPart is the n-th dotted part of a condition's id, or "".
|
|
func idPart(o conditions.Observation, n int) string {
|
|
parts := strings.Split(o.ID, ".")
|
|
if n < len(parts) {
|
|
return parts[n]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
|
func dataWords(o conditions.Observation) (what, module string) {
|
|
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
|
switch {
|
|
case module != "" && item != "":
|
|
return item + " of " + module, module
|
|
case module != "":
|
|
return "data of " + module, module
|
|
}
|
|
return "data", "a module"
|
|
}
|
|
|
|
var plainWordings = map[string]func(conditions.Observation) words{
|
|
// A module's new build whose compositions or grant the controller could not write (composed_keys.go).
|
|
kindNotComposed: worded(func(o conditions.Observation) words {
|
|
m := "a module"
|
|
if p := idPart(o, 0); o.Scope == conditions.ScopeBuild && p != "" {
|
|
m = p // the module: raised in the build's scope, its id the module's name
|
|
}
|
|
return words{Headline: "A new build of " + m + " cannot be delivered",
|
|
Explanation: fmt.Sprintf("A new build of %s was made, and the mesh could not prepare it for the machines, so "+
|
|
"none can be given it yet. It is tried again every half minute; the details say what fails.", m),
|
|
Resolved: "the new build of " + m + " can be delivered"}
|
|
}),
|
|
// The take-in of done keys whose backlog stood still (take_in_consumer.go, novox/hq ADR 0544 §6).
|
|
kindTakeInStalled: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "New builds are not being prepared for the machines",
|
|
Explanation: "The mesh prepares each new build for the machines as it is made, and has prepared none for " +
|
|
"five minutes while builds wait. Until it moves, no new build reaches a machine; the details say how many wait.",
|
|
Resolved: "new builds are being prepared again"}
|
|
}),
|
|
// A bucket of done keys whose take-in consumer could not be made (take_in_consumer.go, novox/hq ADR 0544 §6).
|
|
kindTakeInNotConsumed: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Some new builds are not being prepared",
|
|
Explanation: "The mesh could not start reading one of the two places new builds are announced in, so the " +
|
|
"builds announced there wait until the controller starts again. The details say which and why.",
|
|
Resolved: "the mesh reads where new builds are announced again"}
|
|
}),
|
|
// Config not followed into the compositions (kept_compositions.go, novox/hq ADR 0545).
|
|
kindConfigNotFollowed: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Changes of settings are not reaching the machines",
|
|
Explanation: "The mesh could not read the settings as they change, so a changed setting is not prepared for " +
|
|
"the machines until it can. It tries again every minute; the details say why.",
|
|
Resolved: "changes of settings reach the machines again"}
|
|
}),
|
|
// Machines and their node-engine.
|
|
"silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
w := words{Headline: m + " is not answering",
|
|
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
|
"normal.", m),
|
|
Resolved: m + " answers again"}
|
|
if o.Severity == conditions.Urgent {
|
|
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
|
|
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
|
|
"nothing changes anywhere until it is back.", m)
|
|
}
|
|
return w
|
|
}),
|
|
"sent-not-reported": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has not confirmed a change",
|
|
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
|
|
"It usually catches up by itself.", m, m),
|
|
Resolved: m + " applied the change"}
|
|
}),
|
|
kindApplyFailing: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " is not applying its update",
|
|
Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+
|
|
"The details say what fails.", conditions.Capital(m)),
|
|
Resolved: m + " applies what it is sent again"}
|
|
}),
|
|
"tools-silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Tools on " + m + " are not answering",
|
|
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
|
"the mesh until it does. A machine asleep is normal.", m, m),
|
|
Resolved: "Tools on " + m + " answer again"}
|
|
}),
|
|
"declaration-refused": worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeMesh {
|
|
return words{Headline: "No machine can get new instructions",
|
|
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
|
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
|
"made. Machines keep running what they have.",
|
|
Resolved: "Machines can get new instructions again"}
|
|
}
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " cannot get new instructions",
|
|
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
|
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
|
|
"reaches it. It keeps running what it has.", m),
|
|
Resolved: m + " can get new instructions again"}
|
|
}),
|
|
// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2).
|
|
kindGrantComposesNothing: worded(withheldWords),
|
|
kindAgentCanBecomeRoot: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Sessions on " + m + " could become root",
|
|
Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.",
|
|
Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+
|
|
"can take over the machine without you. The machine cannot show that this holds now, so an answer "+
|
|
"from your phone authorises nothing there until it does.", m),
|
|
Resolved: "Sessions on " + m + " cannot become root again"}
|
|
}),
|
|
"own-address-banned": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has banned the mesh",
|
|
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
|
|
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
|
"mesh is locked out of it.", m),
|
|
Resolved: m + " no longer bans the mesh"}
|
|
}),
|
|
"core-behind": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " runs an old node-engine",
|
|
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
|
|
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
|
|
Resolved: m + " runs the current node-engine"}
|
|
}),
|
|
kindMachineUnits: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Failed services on " + m,
|
|
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
|
|
Explanation: "No module manages them, so the mesh does not repair them.",
|
|
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
|
}),
|
|
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " has a network problem",
|
|
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
|
"route. It often passes again once a network change settles.", conditions.Capital(m)),
|
|
Resolved: m + "'s network is fine again"}
|
|
}),
|
|
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + "'s network settings were changed",
|
|
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
|
"client. Mesh names may not resolve there until it is undone.", m),
|
|
Resolved: m + "'s network settings are the mesh's again"}
|
|
}),
|
|
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: m + " cannot be reached",
|
|
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
|
"asleep is normal.", m),
|
|
Resolved: m + " can be reached again"}
|
|
}),
|
|
// A pending assignment that was not made (novox/hq issue 325, ADR 0261).
|
|
kindPendingEnded: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
module := idPart(o, 1)
|
|
if module == "" {
|
|
module = "a module"
|
|
}
|
|
return words{Headline: module + " was not put on " + m,
|
|
Needs: "decide whether to build it again or take the assignment back; the details say why.",
|
|
Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+
|
|
"register it, so it was not made.", module, m),
|
|
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
|
}),
|
|
// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
|
|
// issue 201).
|
|
kindVaultSilent: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "The vault did not give " + m + " the core's credentials",
|
|
Needs: "check the vault answers (vault_status); the next send asks it again.",
|
|
Explanation: fmt.Sprintf("The bus's and the provisioner's credentials reach %s sealed by the vault, and "+
|
|
"the vault did not answer the last send for them. %s keeps the files it has; a core module not "+
|
|
"there yet is sealed by the controller until the vault holds it.", m, m),
|
|
Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
|
|
}),
|
|
kindFilesNotPut: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "The big files for " + m + " cannot be put on the bus",
|
|
Needs: "look at the bus's permissions for the controller; the details give the reason.",
|
|
Explanation: fmt.Sprintf("The mesh could not put the large files of %s's instructions in their place on the bus, "+
|
|
"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
|
|
Resolved: "Resolved: the big files for " + m + " reach the bus again"}
|
|
}),
|
|
kindBindingKept: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source is held on " + m,
|
|
Needs: "decide where its data lives; the details name both places.",
|
|
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
|
"kept the old one because switching would leave the data behind.", m),
|
|
Resolved: "Resolved: the data source on " + m + " is decided"}
|
|
}),
|
|
kindBindingMoving: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source moves on " + m,
|
|
Needs: "move its data to the new provider before the next push.",
|
|
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
}),
|
|
kindBindingMoved: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "A module's data source moved on " + m,
|
|
Needs: "check that it found its data.",
|
|
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
|
|
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
|
}),
|
|
|
|
// Modules and providers.
|
|
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: conditions.Capital(thing) + " is not working",
|
|
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
|
|
Resolved: conditions.Capital(thing) + " works again"}
|
|
}),
|
|
kindReloginNeeded: worded(func(o conditions.Observation) words {
|
|
// A module's name may hold dots: it is the id without its machine.
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
return reloginWords(orModule(module), machineOr(o, "a machine"), false)
|
|
}),
|
|
kindUsedAsFound: worded(func(o conditions.Observation) words {
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
w := usedAsFoundObservation(orModule(module), machineOr(o, "a machine"), o.Summary, nil)
|
|
return words{Headline: w.Headline, Explanation: w.Explanation, Needs: w.Needs, Resolved: w.Resolved}
|
|
}),
|
|
kindNeedsOperator: worded(func(o conditions.Observation) words {
|
|
// The observation carries the act itself (ADR 0283); these are its words when only the kind is known.
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
node := machineOr(o, "a machine")
|
|
w := needsOperatorWords(orModule(module), node, nil)
|
|
return w
|
|
}),
|
|
kindLeftOut: worded(func(o conditions.Observation) words {
|
|
// The observation carries its own words (novox/hq issue 380); these are its kind's alone.
|
|
module := ""
|
|
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
|
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
|
}
|
|
return leftOutWords(orModule(module), machineOr(o, "a machine"), "")
|
|
}),
|
|
kindProviderFailing: worded(func(o conditions.Observation) words {
|
|
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
|
if consumer == "" {
|
|
consumer = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
|
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
|
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
|
|
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
|
}),
|
|
kindProviderSilent: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: conditions.Capital(thing) + " went quiet",
|
|
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
|
"reporting.", conditions.Capital(thing)),
|
|
Resolved: conditions.Capital(thing) + " reports again"}
|
|
}),
|
|
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: "Retiring data on " + thing + " waits for you",
|
|
Needs: "approve or reject retiring it; the details list what would go.",
|
|
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
|
|
conditions.Capital(thing)),
|
|
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
|
}),
|
|
kindRetireRejected: worded(func(o conditions.Observation) words {
|
|
thing := conditions.ThingWords(o)
|
|
return words{Headline: "Unused data kept on " + thing,
|
|
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
|
|
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
|
}),
|
|
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
|
var where string
|
|
if o.Scope == conditions.ScopeProvider {
|
|
where = conditions.ThingWords(o)
|
|
} else {
|
|
what, _ := dataWords(o)
|
|
where = what + " on " + machineOr(o, "a machine")
|
|
}
|
|
return words{Headline: "Retired data waits for cleanup",
|
|
Needs: "delete it or keep it; nothing is deleted without your word.",
|
|
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
|
|
Resolved: "Resolved: the retired data is dealt with"}
|
|
}),
|
|
|
|
// Data.
|
|
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Data on " + m + " could not be measured",
|
|
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
|
"the data there. The next measurement may settle it.", m),
|
|
Resolved: "Data on " + m + " is measured again"}
|
|
}),
|
|
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " is unprotected",
|
|
Needs: "check the storage it is on.",
|
|
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
|
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is protected again"}
|
|
}),
|
|
kindDataMissing: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " is gone",
|
|
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
|
Needs: "restore it from a backup, or silence this " + FromMeshMCPServer,
|
|
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is back"}
|
|
}),
|
|
kindDataShrank: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
m := machineOr(o, "its machine")
|
|
if idPart(o, 1) == "dataset" {
|
|
what = "a dataset"
|
|
}
|
|
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
|
// No silence from a click: data loss is never waved away from a notification (ADR 0258).
|
|
Needs: "restore the last good copy from a backup, or silence this " + FromMeshMCPServer,
|
|
Explanation: "More than half of what it held is gone within a week.",
|
|
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
|
}),
|
|
kindDataQuiet: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
return words{Headline: conditions.Capital(what) + " stopped changing",
|
|
Needs: "check that what writes it still runs.",
|
|
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
|
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(what) + " is written again"}
|
|
}),
|
|
kindBackupStale: worded(func(o conditions.Observation) words {
|
|
what, _ := dataWords(o)
|
|
m := machineOr(o, "its machine")
|
|
return words{Headline: "No recent backup of " + what,
|
|
Needs: fmt.Sprintf("check that backups run on %s.", m),
|
|
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
|
"changed since would be lost.", conditions.Capital(what), m),
|
|
Resolved: conditions.Capital(what) + " is backed up again"}
|
|
}),
|
|
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Storage on " + m + " is degraded",
|
|
Needs: fmt.Sprintf("check the disks of %s.", m),
|
|
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
|
|
Resolved: "Storage on " + m + " is whole again"}
|
|
}),
|
|
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
|
module := idPart(o, 1)
|
|
if o.Scope == conditions.ScopeProvider {
|
|
module = idPart(o, 2)
|
|
}
|
|
if module == "" {
|
|
module = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
|
Needs: "decide where its data lives: move the data, or move it back.",
|
|
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
|
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
|
|
Resolved: conditions.Capital(module) + " has its data again"}
|
|
}),
|
|
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
|
consumer := idPart(o, 1)
|
|
if consumer == "" {
|
|
consumer = "a module"
|
|
}
|
|
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
|
Needs: "decide which copy to keep.",
|
|
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
|
|
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
|
}),
|
|
|
|
// Deliveries and walks.
|
|
"stalled": worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeDelivery {
|
|
return words{Headline: "A delivery is held too long",
|
|
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
}
|
|
return words{Headline: "A delivery is stuck halfway",
|
|
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
|
Resolved: "Resolved: the delivery moves again"}
|
|
}),
|
|
kindImageMissing: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A module is not updated",
|
|
Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " +
|
|
"was not sent. What runs is untouched.",
|
|
Needs: "build the module again; the details name the image.",
|
|
Resolved: "Resolved: the store holds the image again"}
|
|
}),
|
|
"ask-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A build has no result",
|
|
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
|
Resolved: "Resolved: the build has its result"}
|
|
}),
|
|
"release-held": worded(func(o conditions.Observation) words {
|
|
return releaseHeldWords(nil, o.Also)
|
|
}),
|
|
"release-silent": worded(func(o conditions.Observation) words {
|
|
return releaseSilentWords(o.Also)
|
|
}),
|
|
"facts-stale": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Merge checks use outdated facts",
|
|
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
|
"by themselves.",
|
|
Resolved: "Merge checks use current facts again"}
|
|
}),
|
|
|
|
// The controller and the core.
|
|
"controller-deaf": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The controller stopped listening",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
|
"wait. Changes and repairs do not happen until it recovers.",
|
|
Resolved: "The controller listens again"}
|
|
}),
|
|
"self-check-silent": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh's self-check stopped",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
|
"Problems may go unnoticed until it runs again.",
|
|
Resolved: "The self-check runs again"}
|
|
}),
|
|
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh's watchdogs stopped",
|
|
Needs: "restart the controller if this stays, " + FromMeshMCPServer,
|
|
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
|
|
"missed signals are not noticed.",
|
|
Resolved: "The watchdogs run again"}
|
|
}),
|
|
"lease-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The controller lost its lease",
|
|
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
|
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
|
"not renew it, so two could act at once, or none.",
|
|
Resolved: "The controller holds its lease again"}
|
|
}),
|
|
"lease-split": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Two controllers may be acting",
|
|
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
|
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
|
|
Resolved: "One controller acts again"}
|
|
}),
|
|
"stale-writer": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Outdated instructions are being sent",
|
|
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
|
"were refused, which is harmless.",
|
|
Resolved: "Resolved: no more outdated instructions"}
|
|
}),
|
|
"status-slow": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The mesh answers slowly",
|
|
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
|
Resolved: "The mesh answers quickly again"}
|
|
}),
|
|
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
|
if o.Scope == conditions.ScopeBus {
|
|
return words{Headline: "The bus is not healthy",
|
|
Needs: "check the machine the bus runs on; the details say what fails.",
|
|
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
|
"and tool calls may fail until it recovers.",
|
|
Resolved: "The bus is healthy again"}
|
|
}
|
|
m := machineOr(o, "its machine")
|
|
return words{Headline: "The mesh's own software fails on " + m,
|
|
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
|
|
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
|
}),
|
|
"call-hung": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A tool call is hanging",
|
|
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
|
|
Resolved: "Resolved: the tool call ended"}
|
|
}),
|
|
"probe-failed": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A check of the mesh could not run",
|
|
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
|
|
"again every few minutes.",
|
|
Resolved: "The check runs again"}
|
|
}),
|
|
kindHealersBraked: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "All automatic repairs stopped",
|
|
Needs: "read the details and decide whether the repairs may run again.",
|
|
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
|
"looks like a loop, so all of them stopped.",
|
|
Resolved: "Automatic repairs run again"}
|
|
}),
|
|
"healer-wanted": worded(func(o conditions.Observation) words {
|
|
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
|
if o.Scope != conditions.ScopeMesh {
|
|
cause = "something"
|
|
}
|
|
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
|
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
|
|
"broken now.",
|
|
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
|
}),
|
|
|
|
// The bus and seats.
|
|
"resolver-wrong": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
return words{Headline: "Name lookups fail on " + m,
|
|
Needs: fmt.Sprintf("check that %s is on and online.", m),
|
|
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
|
|
"machines that rely on it may not find each other.", m),
|
|
Resolved: "Name lookups work on " + m + " again"}
|
|
}),
|
|
"holder-silent": worded(func(o conditions.Observation) words {
|
|
m := machineOr(o, "a machine")
|
|
seat := idPart(o, 0)
|
|
if o.Scope != conditions.ScopeSeat {
|
|
seat = "a seat"
|
|
}
|
|
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
|
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
|
"machine asleep is normal.", seat, m),
|
|
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
|
}),
|
|
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus could not be fully set up",
|
|
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
|
|
"again with the next send.",
|
|
Resolved: "The bus is fully set up again"}
|
|
}),
|
|
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus is being upgraded",
|
|
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
|
|
Resolved: "The bus upgrade is done"}
|
|
}),
|
|
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus upgrade failed",
|
|
Needs: "decide whether to put the bus back to the version before; the details say how.",
|
|
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
|
|
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
|
}),
|
|
kindStuck: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Something in a delivery has stopped moving",
|
|
Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " +
|
|
"asked and not answered, a check that stopped, or a part that went offline. Nothing is lost, and it is " +
|
|
"said again until it moves.",
|
|
Resolved: "Resolved: it moves again"}
|
|
}),
|
|
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Messages pile up for a listener",
|
|
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
|
"healer restarts it if it stays.",
|
|
Resolved: "The listener caught up"}
|
|
}),
|
|
"consumer-wrong": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is set up wrong",
|
|
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
|
|
"it up again.",
|
|
Resolved: "The listener is set up right again"}
|
|
}),
|
|
"consumer-lost": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is gone",
|
|
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
|
Resolved: "The listener is back"}
|
|
}),
|
|
"slow-consumer": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener on the bus is too slow",
|
|
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
|
Resolved: "The listener keeps up again"}
|
|
}),
|
|
kindBusFaultsUnread: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Slow clients of the bus cannot be checked",
|
|
Explanation: "The controller could not read what the bus said about its clients' connections, so a " +
|
|
"client the bus cut off would go unnoticed. It tries again at every check.",
|
|
Resolved: "Resolved: the bus's clients are checked again"}
|
|
}),
|
|
"connection-closed": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus cut a client off",
|
|
Explanation: "The bus closed a client's connection because it broke the bus's rules, so what it was " +
|
|
"sending was lost.",
|
|
Resolved: "Resolved: the client keeps to the bus's rules"}
|
|
}),
|
|
kindBusReconnects: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A client keeps losing the bus",
|
|
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
|
"While it reconnects, what it says and what it is asked waits.",
|
|
Resolved: "Resolved: the client stays connected"}
|
|
}),
|
|
"max-deliveries": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A listener gave up on messages",
|
|
Needs: "deliver them again or drop them, from the mesh MCP server.",
|
|
Explanation: "A listener on the bus could not handle messages after several tries, so what they asked " +
|
|
"for was not done. The mesh keeps them until you deliver them again or drop them.",
|
|
Resolved: "Resolved: the messages given up on were delivered again or dropped"}
|
|
}),
|
|
"refused": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "The bus refuses some messages",
|
|
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
|
Resolved: "Resolved: the bus takes the messages again"}
|
|
}),
|
|
kindBucketOrLogFilling: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A module's bucket or log on the bus is filling up",
|
|
Needs: "decide whether to raise its cap or have the module keep less.",
|
|
Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it " +
|
|
"is full, the bus refuses what the module writes there.",
|
|
Resolved: "It has room again"}
|
|
}),
|
|
kindBucketOrLogUnread: worded(func(o conditions.Observation) words {
|
|
return words{Headline: "A module's bucket or log could not be read",
|
|
Explanation: "The controller could not read how full a bucket or log a module keeps on the bus is, so it " +
|
|
"cannot say whether it is filling up. It asks again at its next check.",
|
|
Resolved: "It can be read again"}
|
|
}),
|
|
"stream-wrong": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Part of the bus's storage is wrong",
|
|
Needs: "check the machine the bus runs on; the details say what is missing.",
|
|
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
|
Resolved: "The bus's storage is right again"}
|
|
}),
|
|
"archives-unheld": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Kept builds are about to be cleaned up",
|
|
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
|
|
Resolved: "Resolved: every kept build is claimed"}
|
|
}),
|
|
"archives-missing": worded(func(o conditions.Observation) words {
|
|
return words{Headline: "Kept builds are missing",
|
|
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
|
|
Resolved: "Resolved: the kept builds are back"}
|
|
}),
|
|
}
|
|
|
|
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
|
func humanDuration(d time.Duration) string {
|
|
switch {
|
|
case d < 2*time.Minute:
|
|
return "a minute"
|
|
case d < 2*time.Hour:
|
|
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%d hours", int(d.Hours()))
|
|
}
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
}
|
|
|
|
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
|
func namesWords(names []string, most int) string {
|
|
switch {
|
|
case len(names) == 0:
|
|
return ""
|
|
case len(names) == 1:
|
|
return names[0]
|
|
case len(names) <= most:
|
|
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
|
}
|
|
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
|
}
|
|
|
|
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
|
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
|
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
|
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
|
unit := ""
|
|
for _, r := range rs {
|
|
if strings.Contains(r.Reason, "relogin needed") {
|
|
return reloginNeeds(node)
|
|
}
|
|
if r.Kind == link.KindUnit && unit == "" {
|
|
unit = unitPlainWords(r.Target)
|
|
}
|
|
}
|
|
if unit != "" {
|
|
// Also asked of the operator (moduleActions); the ask's text drops where (askText).
|
|
return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// moduleActions are the answers to a module unhealthy on a machine: restart its failed service there,
|
|
// approved by the operator (novox/hq ADR 0259) — none when the mesh restarts it, or a new login is what it
|
|
// waits for.
|
|
func moduleActions(node string, rs []inventory.ResourceHealth) []conditions.Action {
|
|
for _, r := range rs {
|
|
if strings.Contains(r.Reason, "relogin needed") {
|
|
return nil
|
|
}
|
|
}
|
|
for _, r := range rs {
|
|
if r.Kind != link.KindUnit || r.Target == "" {
|
|
continue
|
|
}
|
|
scope := "system"
|
|
if r.Account != "" {
|
|
scope = "user"
|
|
}
|
|
return []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: node,
|
|
Level: conditions.LevelApprove, Arguments: map[string]string{"unit": r.Target, "scope": scope}}}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP
|
|
// server (the glossary's word; "console" is retired): the answer is not an
|
|
// acknowledgement, so it is given where the operator is known to be the one asking, until answers are
|
|
// authorised (to-be 46 phases 5 and 6).
|
|
const FromMeshMCPServer = "from the mesh MCP server; this notification cannot do it."
|
|
|
|
// releaseHeldWords are the plain words of updates held after a walk failed: which modules wait,
|
|
// on which machines, and where the operator releases them (ADR 0258: a release is not an acknowledgement, so
|
|
// no notification gives it). Raised with the modules and machines (backlogObservation); the kind's fallback
|
|
// knows neither.
|
|
func releaseHeldWords(modules, machines []string) words {
|
|
what := "Some module updates"
|
|
headline := "Updates wait for your release"
|
|
if len(modules) > 0 {
|
|
what = "Updates of " + namesWords(modules, 3)
|
|
if h := what + " wait for your release"; len(h) <= conditions.HeadlineMax {
|
|
headline = h
|
|
} else if h := fmt.Sprintf("%d module updates wait for your release", len(modules)); len(h) <= conditions.HeadlineMax {
|
|
headline = h
|
|
}
|
|
}
|
|
where := ""
|
|
if len(machines) > 0 {
|
|
where = " on " + namesWords(machines, 4)
|
|
}
|
|
return words{Headline: headline,
|
|
Needs: "release them " + FromMeshMCPServer,
|
|
Explanation: fmt.Sprintf("%s%s wait for a person to release them, because the last walk failed."+
|
|
" They are not delivered until then, and stay held if you leave them.", what, where),
|
|
Resolved: "Resolved: the held updates are released"}
|
|
}
|
|
|
|
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
|
|
func releaseSilentWords(machines []string) words {
|
|
where := "Some machines"
|
|
if len(machines) > 0 {
|
|
where = conditions.Capital(namesWords(machines, 4))
|
|
}
|
|
return words{Headline: "Machines did not answer an update",
|
|
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
|
|
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
|
|
"they were left as they were. What they run is not known to be the update.",
|
|
Resolved: "Resolved: the next walk reached them"}
|
|
}
|
|
|
|
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
|
func reloginNeeds(node string) string {
|
|
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
|
}
|
|
|
|
// causeWords is a hand-act's cause as a person says it.
|
|
func causeWords(cause string) string {
|
|
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
|
}
|
|
|
|
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
|
func unitPlainWords(unit string) string {
|
|
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
|
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
|
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
|
if what == "" {
|
|
return strings.ReplaceAll(name, ".", " ")
|
|
}
|
|
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
|
}
|
|
}
|
|
return strings.ReplaceAll(unit, ".", " ")
|
|
}
|
|
|
|
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
|
func resourcePlainWords(r inventory.ResourceHealth) string {
|
|
if r.Kind == link.KindUnit {
|
|
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
|
}
|
|
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return what + " keeps restarting"
|
|
case "down":
|
|
return what + " is not running"
|
|
}
|
|
if r.Check != "" {
|
|
return what + " fails its " + r.Check + " check"
|
|
}
|
|
return what + " is not healthy"
|
|
}
|
|
|
|
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
|
// systemd's result, such as exit-code, or a sentence carrying it).
|
|
func unitReasonWords(reason string) string {
|
|
switch {
|
|
case strings.Contains(reason, "start-limit"):
|
|
return "failed too often to be started again"
|
|
case strings.Contains(reason, "exit-code"):
|
|
return "stopped with an error"
|
|
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
|
return "crashed"
|
|
case strings.Contains(reason, "timeout"):
|
|
return "timed out"
|
|
}
|
|
return "failed"
|
|
}
|
|
|
|
// orModule is a module's name, or "a module" when none is known.
|
|
func orModule(m string) string {
|
|
if m == "" {
|
|
return "a module"
|
|
}
|
|
return m
|
|
}
|