Give every condition a headline, an explanation and a resolved line in plain words
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: --- FAIL: TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt (0.00s)
mesh/delivery-group group feat/plain-notifications rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: --- FAIL: TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt (0.00s)
mesh/delivery-group group feat/plain-notifications rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
The operator could not read the desktop notifications: they carried plan ids, commits, keys and verb syntax. The words the operator reads now travel with the condition, so every channel says them (hq ADR 0253).
This commit is contained in:
@@ -113,6 +113,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
o.Headline, o.Explanation, o.Resolved = stalledWords(l, o)
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -90,8 +90,9 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
|
||||
if len(u.Failed) == 0 {
|
||||
return o, false
|
||||
}
|
||||
var names, said []string
|
||||
var names, said, plain []string
|
||||
for _, f := range u.Failed {
|
||||
plain = append(plain, unitPlainWords(f.Unit))
|
||||
where := ""
|
||||
if f.Scope == "user" {
|
||||
where = " (the account's own manager)"
|
||||
@@ -111,6 +112,14 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
|
||||
"Each is the machine's own: mend or remove it there, or have a module place it",
|
||||
node, len(u.Failed), strings.Join(names, ", "))
|
||||
o.Said = strings.Join(said, "; ")
|
||||
failed := "1 failed service"
|
||||
if len(u.Failed) > 1 {
|
||||
failed = fmt.Sprintf("%d failed services", len(u.Failed))
|
||||
}
|
||||
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
|
||||
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them: mend or "+
|
||||
"remove them on %s.", node, namesWords(plain, 3), node)
|
||||
o.Resolved = "No failed services on " + node + " any more"
|
||||
return o, true
|
||||
}
|
||||
|
||||
|
||||
@@ -167,6 +167,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it, so this is urgent.", len(waiters))
|
||||
}
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
@@ -241,8 +242,9 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
var words, said []string
|
||||
var words, said, plain []string
|
||||
for _, r := range rs {
|
||||
plain = append(plain, resourcePlainWords(r))
|
||||
if r.Kind == link.KindUnit {
|
||||
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
||||
// resource that places it — a package, a file — is evidence.
|
||||
@@ -258,8 +260,12 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; ")}
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; "),
|
||||
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again; if it stays, it "+
|
||||
"needs a look at why it fails.", module, node, namesWords(plain, 3)),
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
package main
|
||||
|
||||
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
||||
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
||||
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
||||
//
|
||||
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
||||
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
||||
// suite is held to having words, and to their being plain (plain_words_test.go).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
type words = conditions.Words
|
||||
|
||||
// worded is a wording that needs only the condition's subject.
|
||||
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
||||
|
||||
func init() {
|
||||
for kind, fn := range plainWordings {
|
||||
conditions.Wording(kind, fn)
|
||||
}
|
||||
}
|
||||
|
||||
// machineOr is the machine a condition concerns, or what to say when none is named.
|
||||
func machineOr(o conditions.Observation, none string) string {
|
||||
if o.Machine != "" {
|
||||
return o.Machine
|
||||
}
|
||||
return none
|
||||
}
|
||||
|
||||
// idPart is the n-th dotted part of a condition's id, or "".
|
||||
func idPart(o conditions.Observation, n int) string {
|
||||
parts := strings.Split(o.ID, ".")
|
||||
if n < len(parts) {
|
||||
return parts[n]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
||||
func dataWords(o conditions.Observation) (what, module string) {
|
||||
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
||||
switch {
|
||||
case module != "" && item != "":
|
||||
return item + " of " + module, module
|
||||
case module != "":
|
||||
return "data of " + module, module
|
||||
}
|
||||
return "data", "a module"
|
||||
}
|
||||
|
||||
var plainWordings = map[string]func(conditions.Observation) words{
|
||||
// Machines and their node-engine.
|
||||
"silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " is not answering",
|
||||
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
||||
"normal; if %s should be on, check that it is powered and online.", m, m),
|
||||
Resolved: m + " answers again"}
|
||||
}),
|
||||
"sent-not-reported": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has not confirmed a change",
|
||||
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them. It "+
|
||||
"usually catches up by itself; if it stays, %s may be stuck.", m, m, m),
|
||||
Resolved: m + " applied the change"}
|
||||
}),
|
||||
"tools-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Tools on " + m + " are not answering",
|
||||
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
||||
"the mesh. A machine asleep is normal.", m, m),
|
||||
Resolved: "Tools on " + m + " answer again"}
|
||||
}),
|
||||
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " waits for new instructions",
|
||||
Explanation: fmt.Sprintf("%s's instructions only work once the next push is made, and none has been made "+
|
||||
"for a while. Nothing is broken; an agent can push to %s.", conditions.Capital(m), m),
|
||||
Resolved: m + " got its new instructions"}
|
||||
}),
|
||||
"declaration-refused": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeMesh {
|
||||
return words{Headline: "No machine can get new instructions",
|
||||
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
||||
"made. Machines keep running what they have. It needs a look now.",
|
||||
Resolved: "Machines can get new instructions again"}
|
||||
}
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot get new instructions",
|
||||
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new reaches "+
|
||||
"%s. It keeps running what it has. It needs a fix in the catalogue.", m, m),
|
||||
Resolved: m + " can get new instructions again"}
|
||||
}),
|
||||
"own-address-banned": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has banned the mesh",
|
||||
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
||||
"mesh is locked out of %s. It needs the ban lifted.", m, m),
|
||||
Resolved: m + " no longer bans the mesh"}
|
||||
}),
|
||||
"core-behind": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " runs an old node-engine",
|
||||
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds, and no "+
|
||||
"delivery is bringing it up to date. It works, but misses the newest fixes.", conditions.Capital(m)),
|
||||
Resolved: m + " runs the current node-engine"}
|
||||
}),
|
||||
kindMachineUnits: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Failed services on " + m,
|
||||
Explanation: fmt.Sprintf("Services on %s that no module manages have failed. They are %s's own, so the "+
|
||||
"mesh does not repair them: mend or remove them there.", m, m),
|
||||
Resolved: "No failed services on " + m + " any more"}
|
||||
}),
|
||||
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has a network problem",
|
||||
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
||||
"route. Other machines may not reach it, or it them.", conditions.Capital(m)),
|
||||
Resolved: m + "'s network is fine again"}
|
||||
}),
|
||||
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + "'s network settings were changed",
|
||||
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
||||
"client. Mesh names may not resolve there until it is undone.", m),
|
||||
Resolved: m + "'s network settings are the mesh's again"}
|
||||
}),
|
||||
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot be reached",
|
||||
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
||||
"asleep is normal; otherwise its tunnel needs a look.", m),
|
||||
Resolved: m + " can be reached again"}
|
||||
}),
|
||||
kindBindingKept: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source is held on " + m,
|
||||
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
||||
"kept the old one because switching would leave the data behind. It needs you to decide.", m),
|
||||
Resolved: "Resolved: the data source on " + m + " is decided"}
|
||||
}),
|
||||
kindBindingMoving: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moves on " + m,
|
||||
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider, and its data is "+
|
||||
"still with the old one. Move the data first.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
kindBindingMoved: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moved on " + m,
|
||||
Explanation: fmt.Sprintf("A module on %s switched to another provider of its data, and its data may "+
|
||||
"still be with the old one. Check it found its data.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
|
||||
// Modules and providers.
|
||||
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " is not working",
|
||||
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again; if it stays, "+
|
||||
"look at why it fails.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " works again"}
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
||||
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
||||
"it up. %s may not work until it does.", conditions.Capital(thing), consumer, conditions.Capital(consumer)),
|
||||
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
||||
}),
|
||||
kindProviderSilent: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " went quiet",
|
||||
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
||||
"reporting. Whether it still fails is not known.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " reports again"}
|
||||
}),
|
||||
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Retiring data on " + thing + " waits for you",
|
||||
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own. It "+
|
||||
"needs you to approve or reject retiring it.", conditions.Capital(thing)),
|
||||
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
||||
}),
|
||||
kindRetireRejected: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Unused data kept on " + thing,
|
||||
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for. Nothing to do; "+
|
||||
"this is a reminder that it is still there.", thing),
|
||||
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
||||
}),
|
||||
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
||||
var where string
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
where = conditions.ThingWords(o)
|
||||
} else {
|
||||
what, _ := dataWords(o)
|
||||
where = what + " on " + machineOr(o, "a machine")
|
||||
}
|
||||
return words{Headline: "Retired data waits for cleanup",
|
||||
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days. It needs you to delete "+
|
||||
"it or keep it.", where),
|
||||
Resolved: "Resolved: the retired data is dealt with"}
|
||||
}),
|
||||
|
||||
// Data.
|
||||
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Data on " + m + " could not be measured",
|
||||
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
||||
"the data there: not that it is safe either.", m),
|
||||
Resolved: "Data on " + m + " is measured again"}
|
||||
}),
|
||||
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is unprotected",
|
||||
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
||||
"the mesh can see. It needs you to check its storage.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is protected again"}
|
||||
}),
|
||||
kindDataMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is gone",
|
||||
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more. If that was not meant, it needs "+
|
||||
"restoring from a backup.", what, machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is back"}
|
||||
}),
|
||||
kindDataShrank: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
m := machineOr(o, "its machine")
|
||||
if idPart(o, 1) == "dataset" {
|
||||
what = "a dataset"
|
||||
}
|
||||
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
||||
Explanation: "More than half of what it held is gone within a week. If you meant that, silence it; if " +
|
||||
"not, restore the last good copy from a backup.",
|
||||
Resolved: "Resolved: the shrinking on " + m + " is explained"}
|
||||
}),
|
||||
kindDataQuiet: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " stopped changing",
|
||||
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
||||
"usual: whatever writes it has stopped.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is written again"}
|
||||
}),
|
||||
kindBackupStale: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: "No recent backup of " + what,
|
||||
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
||||
"changed since would be lost.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is backed up again"}
|
||||
}),
|
||||
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Storage on " + m + " is degraded",
|
||||
Explanation: fmt.Sprintf("The redundant storage on %s lost a disk or is rebuilding. The data on it is "+
|
||||
"less protected until it is whole again.", m),
|
||||
Resolved: "Storage on " + m + " is whole again"}
|
||||
}),
|
||||
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
||||
module := idPart(o, 1)
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
module = idPart(o, 2)
|
||||
}
|
||||
if module == "" {
|
||||
module = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
||||
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
||||
"elsewhere. It needs you: move the data, or move it back.", conditions.Capital(module), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(module) + " has its data again"}
|
||||
}),
|
||||
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
||||
consumer := idPart(o, 1)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
||||
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one. It needs you to "+
|
||||
"decide which to keep.", conditions.Capital(consumer)),
|
||||
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
||||
}),
|
||||
|
||||
// Deliveries and walks.
|
||||
"stalled": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeDelivery {
|
||||
return words{Headline: "A delivery is held too long",
|
||||
Explanation: "A delivery has been held past its time limit. " + conditions.ResolverWords(o),
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}
|
||||
return words{Headline: "A delivery is stuck halfway",
|
||||
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost; if it " +
|
||||
"stays, an agent can look at why.",
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}),
|
||||
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery is waiting to start",
|
||||
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
||||
"out) has not said it may start. Nothing to do yet; an agent can start it by hand.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
"merge-not-acted": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A merge was never picked up",
|
||||
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
||||
"sent. An agent can hand it over again.",
|
||||
Resolved: "Resolved: the merge was picked up"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. What it was for is not delivered until it is " +
|
||||
"built again.",
|
||||
Resolved: "Resolved: the build has its result"}
|
||||
}),
|
||||
kindRolledBack: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
module := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
||||
module = "a module"
|
||||
}
|
||||
if o.Scope == conditions.ScopeCore {
|
||||
return words{Headline: "A core update was put back on " + m,
|
||||
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
||||
"before was put back. It needs a look now.", module, m),
|
||||
Resolved: "Resolved: the core on " + m + " is healthy"}
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
||||
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
||||
"before. The change is not delivered; it needs a fix in its code.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Putting back an update failed on " + m,
|
||||
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before. What runs "+
|
||||
"there may be broken; it needs you now.", m),
|
||||
Resolved: "Resolved: " + m + " runs a good build again"}
|
||||
}),
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Updates wait for your release",
|
||||
Explanation: "Some module updates wait for a person to release them. They are not delivered until you do.",
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date, so a check may pass or " +
|
||||
"fail for the wrong reason.",
|
||||
Resolved: "Merge checks use current facts again"}
|
||||
}),
|
||||
|
||||
// The controller and the core.
|
||||
"controller-deaf": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller stopped listening",
|
||||
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
||||
"wait. Changes and repairs do not happen until it recovers.",
|
||||
Resolved: "The controller listens again"}
|
||||
}),
|
||||
"self-check-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's self-check stopped",
|
||||
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
||||
"Problems may go unnoticed until it runs again.",
|
||||
Resolved: "The self-check runs again"}
|
||||
}),
|
||||
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's watchdogs stopped",
|
||||
Explanation: "The watchdogs, which notice when something expected does not happen, have not run. Missed " +
|
||||
"signals are not being noticed.",
|
||||
Resolved: "The watchdogs run again"}
|
||||
}),
|
||||
"lease-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller lost its lease",
|
||||
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
||||
"not renew it, so two could act at once, or none. It needs a look now.",
|
||||
Resolved: "The controller holds its lease again"}
|
||||
}),
|
||||
"lease-split": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Two controllers may be acting",
|
||||
Explanation: "The lease that lets one controller act at a time does not agree with its record, so two " +
|
||||
"could act at once. It needs a look now.",
|
||||
Resolved: "One controller acts again"}
|
||||
}),
|
||||
"stale-writer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Outdated instructions are being sent",
|
||||
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
||||
"were refused. It is harmless if it stops.",
|
||||
Resolved: "Resolved: no more outdated instructions"}
|
||||
}),
|
||||
"status-slow": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh answers slowly",
|
||||
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
||||
Resolved: "The mesh answers quickly again"}
|
||||
}),
|
||||
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeBus {
|
||||
return words{Headline: "The bus is not healthy",
|
||||
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
||||
"and tool calls may fail until it recovers.",
|
||||
Resolved: "The bus is healthy again"}
|
||||
}
|
||||
m := machineOr(o, "its machine")
|
||||
return words{Headline: "The mesh's own software fails on " + m,
|
||||
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy. What it does there may "+
|
||||
"not happen until it recovers.", m),
|
||||
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
||||
}),
|
||||
"call-hung": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A tool call is hanging",
|
||||
Explanation: "A call to one of the mesh's tools has run far past its time limit. Whatever waits on it is " +
|
||||
"stuck.",
|
||||
Resolved: "Resolved: the tool call ended"}
|
||||
}),
|
||||
"probe-failed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A check of the mesh could not run",
|
||||
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown, which is not " +
|
||||
"the same as fine.",
|
||||
Resolved: "The check runs again"}
|
||||
}),
|
||||
kindHealersBraked: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "All automatic repairs stopped",
|
||||
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
||||
"looks like a loop, so all of them stopped. Nothing repairs itself until this is looked at.",
|
||||
Resolved: "Automatic repairs run again"}
|
||||
}),
|
||||
"healer-wanted": worded(func(o conditions.Observation) words {
|
||||
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
||||
if o.Scope != conditions.ScopeMesh {
|
||||
cause = "something"
|
||||
}
|
||||
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
||||
Explanation: "A person keeps repairing this by hand. A healer (an automatic repair) is wanted for it; " +
|
||||
"nothing is broken now.",
|
||||
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
||||
}),
|
||||
|
||||
// The bus and seats.
|
||||
"resolver-wrong": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Name lookups fail on " + m,
|
||||
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly. "+
|
||||
"Machines that rely on it may not find each other.", m),
|
||||
Resolved: "Name lookups work on " + m + " again"}
|
||||
}),
|
||||
"holder-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
seat := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeSeat {
|
||||
seat = "a seat"
|
||||
}
|
||||
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
||||
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
||||
"machine asleep is normal.", seat, m),
|
||||
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
||||
}),
|
||||
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus could not be fully set up",
|
||||
Explanation: "When instructions were sent, the bus's queues could not all be checked, so a module may " +
|
||||
"not get its messages.",
|
||||
Resolved: "The bus is fully set up again"}
|
||||
}),
|
||||
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus is being upgraded",
|
||||
Explanation: "The mesh's message system is being upgraded; some things pause until it is done. Nothing " +
|
||||
"to do.",
|
||||
Resolved: "The bus upgrade is done"}
|
||||
}),
|
||||
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus upgrade failed",
|
||||
Explanation: "The upgrade of the mesh's message system did not end healthy in time. It needs you: it can " +
|
||||
"be put back to the version before.",
|
||||
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
"healer may restart it.",
|
||||
Resolved: "The listener caught up"}
|
||||
}),
|
||||
"consumer-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is set up wrong",
|
||||
Explanation: "One of the mesh's listeners on the bus is missing or not as the controller defines it, so " +
|
||||
"some messages may reach nobody.",
|
||||
Resolved: "The listener is set up right again"}
|
||||
}),
|
||||
"consumer-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is gone",
|
||||
Explanation: "One of the mesh's listeners is no longer on the bus, so what it should receive reaches " +
|
||||
"nobody.",
|
||||
Resolved: "The listener is back"}
|
||||
}),
|
||||
"slow-consumer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is too slow",
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are dropped or late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
"max-deliveries": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A message could not be handled",
|
||||
Explanation: "The bus gave up on a message after trying to hand it over too many times. What it carried " +
|
||||
"did not happen.",
|
||||
Resolved: "Resolved: messages are handled again"}
|
||||
}),
|
||||
"refused": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus refuses some messages",
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps " +
|
||||
"there may be lost.",
|
||||
Resolved: "The bus's storage is right again"}
|
||||
}),
|
||||
"archives-unheld": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are about to be cleaned up",
|
||||
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them. " +
|
||||
"Nothing to do unless one is still needed.",
|
||||
Resolved: "Resolved: every kept build is claimed"}
|
||||
}),
|
||||
"archives-missing": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are missing",
|
||||
Explanation: "Some builds the mesh relies on are no longer in the store. A machine that needs one cannot " +
|
||||
"get it until it is built again.",
|
||||
Resolved: "Resolved: the kept builds are back"}
|
||||
}),
|
||||
}
|
||||
|
||||
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
||||
func humanDuration(d time.Duration) string {
|
||||
switch {
|
||||
case d < 2*time.Minute:
|
||||
return "a minute"
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
// repoName is a repository as a person says it: its name without its owner.
|
||||
func repoName(repository string) string {
|
||||
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
||||
return repository[i+1:]
|
||||
}
|
||||
return repository
|
||||
}
|
||||
|
||||
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
||||
func namesWords(names []string, most int) string {
|
||||
switch {
|
||||
case len(names) == 0:
|
||||
return ""
|
||||
case len(names) == 1:
|
||||
return names[0]
|
||||
case len(names) <= most:
|
||||
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||
}
|
||||
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
||||
}
|
||||
|
||||
// planModules are the modules a walk moves, sorted.
|
||||
func planModules(p inventory.Plan) []string {
|
||||
out := make([]string, 0, len(p.Modules))
|
||||
for m := range p.Modules {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
||||
// their count and the repository.
|
||||
func deliveryWhat(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2)
|
||||
}
|
||||
if len(modules) > 2 {
|
||||
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
||||
}
|
||||
return repoName(repository)
|
||||
}
|
||||
|
||||
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
||||
func deliveryName(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2) + " delivery"
|
||||
}
|
||||
return repoName(repository) + " delivery"
|
||||
}
|
||||
|
||||
// waitingAct is what the operator does about a walk waiting for its delivery's word.
|
||||
func waitingAct(severity conditions.Severity) string {
|
||||
if severity == conditions.Urgent {
|
||||
return "Mesh-delivery may be broken. It needs you: have an agent look at mesh-delivery, or start the " +
|
||||
"delivery by hand."
|
||||
}
|
||||
return fmt.Sprintf("Nothing to do yet; it becomes urgent after %s. To start it now, have an agent start it by hand.",
|
||||
humanDuration(waitUrgentAfter))
|
||||
}
|
||||
|
||||
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
||||
func stalledWords(l stalledLine, o conditions.Observation) (string, string, string) {
|
||||
repository, _, _ := strings.Cut(l.ID, "@")
|
||||
name := repoName(repository)
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
}
|
||||
long := "too long"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
act := conditions.ResolverWords(o)
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
act = "It waits for you: release it, or stop it."
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
fmt.Sprintf("A delivery of %s has been %s %s, past its limit. %s", name, held, long, act),
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held)
|
||||
}
|
||||
|
||||
// causeWords is a hand-act's cause as a person says it.
|
||||
func causeWords(cause string) string {
|
||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
}
|
||||
|
||||
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
||||
func unitPlainWords(unit string) string {
|
||||
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
||||
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
||||
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
||||
if what == "" {
|
||||
return strings.ReplaceAll(name, ".", " ")
|
||||
}
|
||||
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(unit, ".", " ")
|
||||
}
|
||||
|
||||
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
||||
func resourcePlainWords(r inventory.ResourceHealth) string {
|
||||
if r.Kind == link.KindUnit {
|
||||
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
||||
}
|
||||
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return what + " keeps restarting"
|
||||
case "down":
|
||||
return what + " is not running"
|
||||
}
|
||||
if r.Check != "" {
|
||||
return what + " fails its " + r.Check + " check"
|
||||
}
|
||||
return what + " is not healthy"
|
||||
}
|
||||
|
||||
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
||||
// systemd's result, such as exit-code, or a sentence carrying it).
|
||||
func unitReasonWords(reason string) string {
|
||||
switch {
|
||||
case strings.Contains(reason, "start-limit"):
|
||||
return "failed too often to be started again"
|
||||
case strings.Contains(reason, "exit-code"):
|
||||
return "stopped with an error"
|
||||
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
||||
return "crashed"
|
||||
case strings.Contains(reason, "timeout"):
|
||||
return "timed out"
|
||||
}
|
||||
return "failed"
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The conditions the operator read on 2026-10-08, raised again from the same facts (novox/hq ADR 0253):
|
||||
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
|
||||
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
|
||||
|
||||
// plainExample checks one finding's words: plain, within their bounds, and saying what the operator needs.
|
||||
func plainExample(t *testing.T, o conditions.Observation, headline string, says ...string) {
|
||||
t.Helper()
|
||||
machines := append([]string{o.Machine}, o.Also...)
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
|
||||
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
||||
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
|
||||
}
|
||||
if o.Headline != headline {
|
||||
t.Errorf("%s: headline %q, want %q", o.Key(), o.Headline, headline)
|
||||
}
|
||||
for _, s := range says {
|
||||
if !strings.Contains(o.Explanation+" | "+o.Resolved, s) {
|
||||
t.Errorf("%s: the words do not say %q:\n %s | %s", o.Key(), s, o.Explanation, o.Resolved)
|
||||
}
|
||||
}
|
||||
t.Logf("\nBEFORE: %s\nAFTER: %s\n %s\nCLEARS: %s", o.Summary, o.Headline, o.Explanation, o.Resolved)
|
||||
}
|
||||
|
||||
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
|
||||
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
|
||||
// `plans go plan-1791454185265004861 --why …` starts it by hand".
|
||||
func TestADeliveryWaitingIsSaidByItsModulesAndWhatToDo(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
|
||||
awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start", "56 minutes", "mesh-delivery (the module that decides",
|
||||
"Nothing to do yet", "openrazer delivery no longer waiting")
|
||||
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
|
||||
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
|
||||
}
|
||||
|
||||
// Past four hours it is urgent, and says it needs the operator.
|
||||
f.waits[0].since = now.Add(-5 * time.Hour)
|
||||
got = watchWaits(f)
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start", "It needs you")
|
||||
|
||||
// Many modules are counted, not listed in the headline.
|
||||
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
||||
got = watchWaits(f)
|
||||
plainExample(t, got[0], "mesh-catalog delivery waiting to start", "4 modules of mesh-catalog")
|
||||
}
|
||||
|
||||
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
||||
// account's own service manager (exit-code)".
|
||||
func TestAModuleUnhealthyNamesItsServiceInWords(t *testing.T) {
|
||||
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
||||
plainExample(t, o, "openrazer not working on g14", "is not healthy: its service openrazer-daemon stopped with an error",
|
||||
"openrazer works again on g14")
|
||||
}
|
||||
|
||||
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
|
||||
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). Each is the
|
||||
// machine's own: mend or remove it there, or have a module place it".
|
||||
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
|
||||
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
|
||||
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
|
||||
{Unit: "greenclip.service", Scope: "user", Result: "exit-code"}}})
|
||||
if !raise {
|
||||
t.Fatal("not raised")
|
||||
}
|
||||
plainExample(t, o, "3 failed services on shanks",
|
||||
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed", "mend or remove them on shanks")
|
||||
}
|
||||
|
||||
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools,
|
||||
// through the mesh-controller seat: a healer is wanted for it".
|
||||
func TestAHealerWantedSaysNothingIsBroken(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
for i := 0; i < 35; i++ {
|
||||
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), "push")
|
||||
a.Verb, a.By = "push", "g14.node-tools, through the mesh-controller seat"
|
||||
f.handActs = append(f.handActs, a)
|
||||
}
|
||||
got := watchHandActs(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
plainExample(t, got[0], "Push keeps being fixed by hand", "35 times in 14 days", "nothing is broken now")
|
||||
}
|
||||
|
||||
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
|
||||
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
|
||||
func TestADeliveryHeldSaysItWaitsForTheOperator(t *testing.T) {
|
||||
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
||||
plainExample(t, got[0], "Delivery of hq held for 36 hours", "It waits for you: release it, or stop it.")
|
||||
}
|
||||
|
||||
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
||||
// shape it is raised with.
|
||||
func TestEveryWordingIsPlain(t *testing.T) {
|
||||
subjects := []conditions.Observation{
|
||||
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
|
||||
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
|
||||
{Scope: conditions.ScopeProvider, ID: "postgres.anchor.app_db", Machine: "anchor"},
|
||||
{Scope: conditions.ScopePlan, ID: "plan-1791454185265004861"},
|
||||
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
|
||||
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
|
||||
{Scope: conditions.ScopeBus, ID: "mesh"},
|
||||
{Scope: conditions.ScopeSeat, ID: "node-backup.ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMesh, ID: "hand-acts.disk-load"},
|
||||
{Scope: conditions.ScopeBuild, ID: "openrazer.g14", Machine: "g14"},
|
||||
}
|
||||
for kind, fn := range plainWordings {
|
||||
if !conditions.Worded(kind) {
|
||||
t.Errorf("%s is not registered", kind)
|
||||
}
|
||||
for _, s := range subjects {
|
||||
s.Kind = kind
|
||||
w := fn(s)
|
||||
if why, ok := conditions.PlainWords(w, s.Machine); !ok {
|
||||
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,20 @@ func TestMain(m *testing.M) {
|
||||
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
|
||||
unsaid.note(o, field, r)
|
||||
}
|
||||
conditions.Unworded = func(o conditions.Observation, why string) {
|
||||
if o.Source != "test" { // a kind a test makes up to exercise the keeper
|
||||
unworded.add(o, why)
|
||||
}
|
||||
}
|
||||
code := m.Run()
|
||||
if said := unworded.all(); len(said) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+
|
||||
"words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+
|
||||
"reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n "))
|
||||
if code == 0 {
|
||||
code = 1
|
||||
}
|
||||
}
|
||||
if said := unsaid.all(); len(said) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
|
||||
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
|
||||
@@ -74,6 +87,34 @@ func (u *unsayable) all() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// unworded collects, across the suite, every finding said in borrowed words (ADR 0253).
|
||||
var unworded wordless
|
||||
|
||||
type wordless struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]bool
|
||||
}
|
||||
|
||||
func (u *wordless) add(o conditions.Observation, why string) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
if u.seen == nil {
|
||||
u.seen = map[string]bool{}
|
||||
}
|
||||
u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true
|
||||
}
|
||||
|
||||
func (u *wordless) all() []string {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
var out []string
|
||||
for s := range u.seen {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
|
||||
// that reads a producer's findings without raising them.
|
||||
func linted(obs []conditions.Observation) []conditions.Observation {
|
||||
@@ -85,6 +126,18 @@ func linted(obs []conditions.Observation) []conditions.Observation {
|
||||
if r, ok := outward.Check(o.Summary, machines...); !ok {
|
||||
unsaid.note(o, "summary", r)
|
||||
}
|
||||
switch {
|
||||
case o.Headline != "":
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
|
||||
if w.Resolved == "" {
|
||||
w.Resolved = "Resolved"
|
||||
}
|
||||
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
||||
unworded.add(o, "its source's words are not plain: "+why)
|
||||
}
|
||||
case o.Kind != "" && !conditions.Worded(o.Kind):
|
||||
unworded.add(o, "the kind "+o.Kind+" has no plain words")
|
||||
}
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
@@ -322,6 +322,10 @@ func watchPlans(f *signalFacts) []conditions.Observation {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||
Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway",
|
||||
Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+
|
||||
"usual. Nothing is lost; if it stays, an agent can look at why.", humanDuration(f.now.Sub(p.entered))),
|
||||
Resolved: "Delivery of " + repoName(p.repository) + " moves again",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||
@@ -353,7 +357,12 @@ func watchWaits(f *signalFacts) []conditions.Observation {
|
||||
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
|
||||
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
|
||||
short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
|
||||
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits)})
|
||||
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
|
||||
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
|
||||
Explanation: fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the "+
|
||||
"module that decides when a delivery goes out) to let it start. %s", deliveryWhat(w.modules, w.repository),
|
||||
humanDuration(in), waitingAct(severity)),
|
||||
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -681,7 +690,12 @@ func watchHandActs(f *signalFacts) []conditions.Observation {
|
||||
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause,
|
||||
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
|
||||
Said: strings.Join(acts, "; ")})
|
||||
Said: strings.Join(acts, "; "),
|
||||
Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand",
|
||||
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days. A healer (an automatic repair) "+
|
||||
"is wanted for it; nothing is broken now.", causeWords(cause), repeated[cause],
|
||||
int(handActsWithin.Hours()/24)),
|
||||
Resolved: "Resolved: no more hand repairs of " + causeWords(cause)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -150,6 +150,8 @@ type planFacts struct {
|
||||
type waitFacts struct {
|
||||
id, repository, commit, awaits string
|
||||
since time.Time
|
||||
// modules are the modules the walk moves, as a person names the delivery.
|
||||
modules []string
|
||||
}
|
||||
|
||||
type loopFacts struct {
|
||||
@@ -451,7 +453,7 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
|
||||
// S16's, whatever mesh-delivery says or does not say.
|
||||
if p.Waiting() {
|
||||
waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit,
|
||||
awaits: p.Delivery.Awaits, since: p.Created})
|
||||
awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p)})
|
||||
continue
|
||||
}
|
||||
_, paused := pausedWaiting(p, pause, now)
|
||||
|
||||
@@ -125,8 +125,15 @@ type Condition struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
// Summary is one line in the mesh's words.
|
||||
// Summary is one line in the mesh's words, for whoever looks closer: it may name plans, commits and
|
||||
// the verbs that act.
|
||||
Summary string `json:"summary"`
|
||||
// Headline, Explanation and Resolved are what the operator reads, in plain words (novox/hq ADR 0253,
|
||||
// plain.go): a few words naming the thing and what is wrong; one or two sentences on what happened,
|
||||
// what it means and whether to act; the one line said when it clears.
|
||||
Headline string `json:"headline"`
|
||||
Explanation string `json:"explanation"`
|
||||
Resolved string `json:"resolved"`
|
||||
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||
@@ -175,6 +182,11 @@ type Observation struct {
|
||||
Also []string
|
||||
Severity Severity
|
||||
Summary string
|
||||
// Headline, Explanation and Resolved are the plain words the operator reads (plain.go). Left empty,
|
||||
// the wording registered for Kind says them.
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
|
||||
// here, never in Summary**: an address, a socket's error, a path or a name with its domain is
|
||||
// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
package conditions
|
||||
|
||||
// Plain words (novox/hq ADR 0253): **a condition carries what the operator reads, in plain words, beside
|
||||
// what an agent reads.** The summary is one line in the mesh's words for whoever looks closer — it names
|
||||
// plans, commits and the verbs that act, and stays so. The operator reads a notification between other
|
||||
// work: a popup that said "the walk of novox/mesh-catalog a6385479 has waited 56m0s for mesh-delivery's
|
||||
// word to start: `mesh-delivery.show` …" was a wall of identifiers nobody could act on. So every condition
|
||||
// also carries:
|
||||
//
|
||||
// - a **headline**: a few plain words naming the thing and what is wrong ("openrazer not running on
|
||||
// g14"), the title of every message about it;
|
||||
// - an **explanation**: one or two plain sentences — what happened, what it means for the operator,
|
||||
// and whether they need to do anything;
|
||||
// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14").
|
||||
//
|
||||
// They are made here, where the condition is made, so every channel gets them — the desktop today,
|
||||
// others later — and none has to guess what a key means. A producer may say them itself; otherwise the
|
||||
// wording registered for its kind says them; otherwise a plain sentence made from its scope does, and a
|
||||
// test suite is told (Unworded) so the kind gets words of its own. Each is held to the plain rule (Plain):
|
||||
// no identifiers, hashes, keys, verb syntax, markup or clock times — a message names no time of its own,
|
||||
// since the channel says when, in the operator's time.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/outward"
|
||||
)
|
||||
|
||||
// Words are what the operator reads of a condition.
|
||||
type Words struct {
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
}
|
||||
|
||||
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
|
||||
const (
|
||||
HeadlineMax = 60
|
||||
ExplanationMax = 360
|
||||
)
|
||||
|
||||
var (
|
||||
wordingsMu sync.RWMutex
|
||||
wordings = map[string]func(Observation) Words{}
|
||||
)
|
||||
|
||||
// Wording registers the plain words of one condition kind.
|
||||
func Wording(kind string, words func(Observation) Words) {
|
||||
wordingsMu.Lock()
|
||||
defer wordingsMu.Unlock()
|
||||
wordings[kind] = words
|
||||
}
|
||||
|
||||
// Worded says whether a kind has words of its own.
|
||||
func Worded(kind string) bool {
|
||||
wordingsMu.RLock()
|
||||
defer wordingsMu.RUnlock()
|
||||
_, ok := wordings[kind]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Unworded is told of every observation said in borrowed words: its kind has none registered, or the
|
||||
// words it was given break the plain rule. The keeper says it plainly anyway; a test suite sets this to
|
||||
// fail the producer.
|
||||
var Unworded func(o Observation, why string)
|
||||
|
||||
// The plain rule's shapes.
|
||||
var (
|
||||
hexID = regexp.MustCompile(`\b[0-9a-f]{7,40}\b`)
|
||||
numberedID = regexp.MustCompile(`\b[a-z]+-[0-9]{6,}\b`)
|
||||
dotted = regexp.MustCompile(`\b[a-z][a-z0-9_-]*\.[a-z][a-z0-9_-]*\b`)
|
||||
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
|
||||
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
|
||||
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
|
||||
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
|
||||
)
|
||||
|
||||
// Plain says whether text is plain words the operator reads at a glance, and if not, what is not: an
|
||||
// identifier (a hash, a numbered id, a dotted name such as a key or a verb), a Go duration, a clock time
|
||||
// or date (the channel says when, in the operator's time), a command's flag, markup, a line break, or
|
||||
// anything the operator's channel withholds (ADR 0234 §6). machines are names that may appear.
|
||||
func Plain(text string, machines ...string) (string, bool) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return "nothing is said", false
|
||||
}
|
||||
if strings.ContainsAny(text, "\n\r\t") {
|
||||
return "a line break", false
|
||||
}
|
||||
if m := markup.FindString(text); m != "" {
|
||||
return "markup (" + m + ")", false
|
||||
}
|
||||
for _, shape := range []struct {
|
||||
re *regexp.Regexp
|
||||
what string
|
||||
}{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"},
|
||||
{clockTime, "a clock time or date"}, {flag, "a command's flag"}} {
|
||||
if m := shape.re.FindString(text); m != "" {
|
||||
if shape.re == hexID && !strings.ContainsAny(m, "0123456789") {
|
||||
continue // a word of letters a to f only
|
||||
}
|
||||
return shape.what + " (" + strings.TrimSpace(m) + ")", false
|
||||
}
|
||||
}
|
||||
if r, ok := outward.Check(text, machines...); !ok {
|
||||
return r.String(), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// PlainWords checks all three, with their bounds.
|
||||
func PlainWords(w Words, machines ...string) (string, bool) {
|
||||
if why, ok := Plain(w.Headline, machines...); !ok {
|
||||
return "headline: " + why, false
|
||||
}
|
||||
if len(w.Headline) > HeadlineMax {
|
||||
return fmt.Sprintf("headline: longer than %d characters", HeadlineMax), false
|
||||
}
|
||||
if why, ok := Plain(w.Explanation, machines...); !ok {
|
||||
return "explanation: " + why, false
|
||||
}
|
||||
if len(w.Explanation) > ExplanationMax {
|
||||
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
|
||||
}
|
||||
if why, ok := Plain(w.Resolved, machines...); !ok {
|
||||
return "resolved: " + why, false
|
||||
}
|
||||
if len(w.Resolved) > HeadlineMax+20 {
|
||||
return fmt.Sprintf("resolved: longer than %d characters", HeadlineMax+20), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// plainly gives an observation its plain words: its own, its kind's, or its scope's.
|
||||
func plainly(o Observation) Observation {
|
||||
machines := append([]string{o.Machine}, o.Also...)
|
||||
given := Words{o.Headline, o.Explanation, o.Resolved}
|
||||
var w Words
|
||||
from := ""
|
||||
switch {
|
||||
case given.Headline != "":
|
||||
w, from = given, "its source"
|
||||
default:
|
||||
wordingsMu.RLock()
|
||||
fn := wordings[o.Kind]
|
||||
wordingsMu.RUnlock()
|
||||
if fn != nil {
|
||||
w, from = fn(o), "the wording of "+o.Kind
|
||||
if given.Explanation != "" {
|
||||
w.Explanation = given.Explanation
|
||||
}
|
||||
if given.Resolved != "" {
|
||||
w.Resolved = given.Resolved
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.Headline != "" && w.Resolved == "" {
|
||||
w.Resolved = "Resolved: " + lowerFirst(w.Headline)
|
||||
}
|
||||
if from == "" {
|
||||
if Unworded != nil {
|
||||
Unworded(o, "the kind "+o.Kind+" has no plain words")
|
||||
}
|
||||
w = scopeWords(o)
|
||||
} else if why, ok := PlainWords(w, machines...); !ok {
|
||||
if Unworded != nil {
|
||||
Unworded(o, from+" is not plain: "+why)
|
||||
}
|
||||
w = scopeWords(o)
|
||||
}
|
||||
o.Headline, o.Explanation, o.Resolved = w.Headline, w.Explanation, w.Resolved
|
||||
return o
|
||||
}
|
||||
|
||||
// Escalated is the sentence a condition's explanation ends with once a healer gave up on it.
|
||||
const Escalated = "The mesh tried to repair it and could not: it needs you now."
|
||||
|
||||
// escalatedWords says in the explanation that a healer gave up, whatever the words said before.
|
||||
func escalatedWords(c *Condition) {
|
||||
if c.Escalated() && !strings.HasSuffix(c.Explanation, Escalated) {
|
||||
c.Explanation = strings.TrimSpace(c.Explanation + " " + Escalated)
|
||||
}
|
||||
}
|
||||
|
||||
// scopeWords is what is said of a kind with no words of its own: what kind of thing, and which one
|
||||
// where its name is a name (a machine, a module), never an id.
|
||||
func scopeWords(o Observation) Words {
|
||||
thing := ThingWords(o)
|
||||
what := strings.ReplaceAll(o.Kind, "-", " ")
|
||||
w := Words{
|
||||
Headline: Capital(thing) + " needs a look",
|
||||
Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s. %s", what, thing, ResolverWords(o)),
|
||||
Resolved: "Resolved: " + thing + " is fine again",
|
||||
}
|
||||
if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok {
|
||||
w = Words{Headline: "Something in the mesh needs a look",
|
||||
Explanation: "The mesh noticed a problem it has no plain words for yet. " + ResolverWords(o),
|
||||
Resolved: "Resolved: the mesh is fine again"}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// ThingWords is what a condition is about, in plain words: "the machine ace", "openrazer on g14", "a
|
||||
// delivery". An id that is not a name (a plan's, a delivery's, a call's) is never said.
|
||||
func ThingWords(o Observation) string {
|
||||
switch o.Scope {
|
||||
case ScopeMachine:
|
||||
if o.Machine != "" {
|
||||
return o.Machine
|
||||
}
|
||||
return "a machine"
|
||||
case ScopeModule:
|
||||
module := strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
if o.Machine != "" && module != "" && !strings.Contains(module, ".") {
|
||||
return module + " on " + o.Machine
|
||||
}
|
||||
return "a module"
|
||||
case ScopeProvider:
|
||||
if module, _, _ := strings.Cut(o.ID, "."); module != "" && o.Machine != "" {
|
||||
return module + " on " + o.Machine
|
||||
}
|
||||
return "a provider"
|
||||
case ScopePlan:
|
||||
return "a walk"
|
||||
case ScopeDelivery:
|
||||
return "a delivery"
|
||||
case ScopeBuild:
|
||||
return "a build"
|
||||
case ScopeCall:
|
||||
return "a tool call"
|
||||
case ScopeMerge:
|
||||
return "a merge"
|
||||
case ScopeSeat:
|
||||
return "a seat's holder"
|
||||
case ScopeBus:
|
||||
return "the bus"
|
||||
case ScopeCore:
|
||||
return "the controller"
|
||||
case ScopeProbe:
|
||||
return "the self-check"
|
||||
}
|
||||
return "the mesh"
|
||||
}
|
||||
|
||||
// ResolverWords says whether the operator needs to act, from who resolves it.
|
||||
func ResolverWords(o Observation) string {
|
||||
switch {
|
||||
case o.Resolver == ResolverOperator:
|
||||
return "It needs you: the mesh does not repair this by itself."
|
||||
case strings.HasPrefix(o.Resolver, "healer:"):
|
||||
return "A healer is working on it; nothing to do unless it stays."
|
||||
case o.Severity == Urgent:
|
||||
return "It clears by itself once it is fixed, but needs a look now."
|
||||
}
|
||||
return "Nothing to do yet: it clears by itself once it is fixed."
|
||||
}
|
||||
|
||||
// Capital is s with its first letter upper case.
|
||||
func Capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
|
||||
func lowerFirst(s string) string {
|
||||
if s == "" || (len(s) > 1 && strings.ToUpper(s[:2]) == s[:2]) {
|
||||
return s
|
||||
}
|
||||
return strings.ToLower(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The plain rule** (novox/hq ADR 0253): what the operator reads names no identifier, no code and no time
|
||||
// of its own; the words conditions are made of pass.
|
||||
func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) {
|
||||
refused := map[string]string{
|
||||
"the walk of novox/mesh-catalog a6385479 waits": "a hash",
|
||||
"plan plan-1791454185265004861 is waiting": "a hash",
|
||||
"see `mesh-delivery.show` for why": "markup",
|
||||
"call mesh-delivery.show for why": "a dotted name",
|
||||
"key plan.x.waiting": "a dotted name",
|
||||
"waited 56m0s": "a duration in code",
|
||||
"since 10:40": "a clock time",
|
||||
"since 2026-10-08": "a clock time",
|
||||
"raised at noon UTC": "a clock time",
|
||||
"start it with plans go --why": "a command's flag",
|
||||
"one line\nand another": "a line break",
|
||||
"the resolver at 10.77.0.1 is silent": "",
|
||||
"": "nothing",
|
||||
"**bold**": "markup",
|
||||
"storage-media.mount failed": "a dotted name",
|
||||
"the store at /var/lib/mesh is full": "",
|
||||
"a commit 0123abc landed": "a hash",
|
||||
}
|
||||
for text, want := range refused {
|
||||
why, ok := Plain(text, "g14")
|
||||
if ok {
|
||||
t.Errorf("%q passed", text)
|
||||
continue
|
||||
}
|
||||
if want != "" && !strings.HasPrefix(why, want) {
|
||||
t.Errorf("%q: refused for %q, want %q", text, why, want)
|
||||
}
|
||||
}
|
||||
for _, text := range []string{
|
||||
"openrazer delivery waiting to start",
|
||||
"The change to openrazer is merged and built, and has waited 56 minutes for mesh-delivery to let it start.",
|
||||
"3 failed services on shanks",
|
||||
"On shanks, mnt-recalbox (a mount), storage-media (a mount) and greenclip failed.",
|
||||
"Name lookups fail on g14",
|
||||
"Retired data of postgres on anchor waits for app_db",
|
||||
"Push keeps being fixed by hand",
|
||||
} {
|
||||
if why, ok := Plain(text, "g14", "shanks", "anchor"); !ok {
|
||||
t.Errorf("%q refused: %s", text, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A condition carries its plain words**: a producer's own; else its kind's; else its scope's — and the
|
||||
// event says them beside the summary, for every channel.
|
||||
func TestAConditionCarriesItsPlainWords(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
before := Unworded
|
||||
var unworded []string
|
||||
Unworded = func(o Observation, why string) { unworded = append(unworded, why) }
|
||||
t.Cleanup(func() { Unworded = before })
|
||||
|
||||
Wording("test-silent", func(o Observation) Words {
|
||||
return Words{Headline: o.Machine + " is not answering", Explanation: "The mesh has not heard from it.",
|
||||
Resolved: o.Machine + " answers again"}
|
||||
})
|
||||
c, err := k.Observe(t.Context(), Observation{Scope: ScopeMachine, ID: "ace", Kind: "test-silent", Machine: "ace",
|
||||
Severity: Warning, Source: "S1", Summary: "ace has not been heard from since 12:00 UTC (bound 3m0s)"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Headline != "ace is not answering" || c.Resolved != "ace answers again" || c.Explanation == "" {
|
||||
t.Fatalf("the kind's words were not kept: %+v", c)
|
||||
}
|
||||
if c.Summary != "ace has not been heard from since 12:00 UTC (bound 3m0s)" {
|
||||
t.Errorf("the summary was changed: %q", c.Summary)
|
||||
}
|
||||
said := settled(t, told, 1)
|
||||
body, _ := json.Marshal(said[0])
|
||||
for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"The mesh has not heard from it."`,
|
||||
`"resolved":"ace answers again"`} {
|
||||
if !strings.Contains(string(body), field) {
|
||||
t.Errorf("the event does not carry %s: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// A producer's own words win; a resolved line it leaves out is made from its headline.
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Kind: "test-silent",
|
||||
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy",
|
||||
Headline: "openrazer not working on g14", Explanation: "Its service stopped with an error."})
|
||||
if c.Headline != "openrazer not working on g14" || c.Resolved != "Resolved: openrazer not working on g14" {
|
||||
t.Errorf("the producer's words: %+v", c)
|
||||
}
|
||||
if len(unworded) != 0 {
|
||||
t.Errorf("words that were plain were reported: %v", unworded)
|
||||
}
|
||||
|
||||
// Words that are not plain, and a kind with none, are said from the scope — and reported.
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded",
|
||||
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"})
|
||||
if c.Headline != "Openrazer on g14 needs a look" || !strings.Contains(c.Explanation, "Nothing to do yet") {
|
||||
t.Errorf("the scope's words: %+v", c)
|
||||
}
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent",
|
||||
Severity: Warning, Source: "S16", Summary: "the walk waits", Headline: "plan-1791454185265004861 waits"})
|
||||
if strings.Contains(c.Headline, "plan-") || c.Headline != "A walk needs a look" {
|
||||
t.Errorf("an id reached the headline: %+v", c)
|
||||
}
|
||||
if len(unworded) != 2 || !strings.Contains(unworded[0], "no plain words") || !strings.Contains(unworded[1], "not plain") {
|
||||
t.Errorf("what was said in borrowed words was not reported: %v", unworded)
|
||||
}
|
||||
}
|
||||
|
||||
// **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the
|
||||
// budget is spent, the operator is needed.
|
||||
func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
|
||||
k, _, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
|
||||
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "Nothing to do yet."}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _, err := k.Escalate(ctx, o.Key(), Attempt{What: "budget spent", Outcome: "escalated", By: "healer H1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(c.Explanation, Escalated) {
|
||||
t.Fatalf("escalated: %q", c.Explanation)
|
||||
}
|
||||
c, _ = k.Observe(ctx, o)
|
||||
if strings.Count(c.Explanation, Escalated) != 1 {
|
||||
t.Errorf("seen again after escalation: %q", c.Explanation)
|
||||
}
|
||||
}
|
||||
@@ -175,6 +175,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
return Condition{}, err
|
||||
}
|
||||
o = k.sayable(o)
|
||||
o = plainly(o)
|
||||
key := o.Key()
|
||||
for i := 0; i < tries; i++ {
|
||||
now := k.now().UTC()
|
||||
@@ -188,7 +189,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
}
|
||||
if !found {
|
||||
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||
Resolver: orSelf(o.Resolver)}
|
||||
change := ChangeRaised
|
||||
@@ -238,6 +240,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
// The kind as the source says it now: a source that gave the same key a kind of its own since
|
||||
// (a probe's finding split out for a healer) is read by that kind from its next observation.
|
||||
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
|
||||
c.Headline, c.Explanation, c.Resolved = o.Headline, o.Explanation, o.Resolved
|
||||
escalatedWords(&c)
|
||||
if o.Machine != "" {
|
||||
c.Subject.Machine = o.Machine
|
||||
}
|
||||
@@ -335,6 +339,7 @@ func (k *Keeper) Escalate(ctx context.Context, key string, a Attempt) (Condition
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
|
||||
c.Resolver = ResolverOperator
|
||||
}
|
||||
escalatedWords(c)
|
||||
return changes
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user