Open every explanation with what the operator needs to do, and offer the answers
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/plain-notifications delivered: every member is delivered

The operator could not tell from a notification whether to act, and was told to
have an agent do it. Each condition now says "Nothing for you to do." or
"Needs you:" with one thing they can do themselves, and carries the actions the
operator channel performs when chosen (hq ADR 0253).
This commit is contained in:
jochen
2026-10-08 13:58:53 +02:00
parent 7df72edd2b
commit 1a4305213d
10 changed files with 430 additions and 207 deletions
+1 -1
View File
@@ -113,7 +113,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
if l.operatorsOnly() {
o.Resolver = conditions.ResolverOperator
}
o.Headline, o.Explanation, o.Resolved = stalledWords(l, o)
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
out = append(out, o)
}
return out
+4 -2
View File
@@ -117,8 +117,10 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
failed = fmt.Sprintf("%d failed services", len(u.Failed))
}
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them: mend or "+
"remove them on %s.", node, namesWords(plain, 3), node)
o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
node, namesWords(plain, 3))
o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
o.Resolved = "No failed services on " + node + " any more"
return o, true
}
+6 -3
View File
@@ -167,7 +167,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it, so this is urgent.", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
}
}
seen[o.Key()] = true
@@ -243,6 +243,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
var words, said, plain []string
needs, actions := moduleNeeds(node, rs)
for _, r := range rs {
plain = append(plain, resourcePlainWords(r))
if r.Kind == link.KindUnit {
@@ -263,8 +264,10 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
Said: strings.Join(said, "; "),
Headline: fmt.Sprintf("%s not working on %s", module, node),
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again; if it stays, it "+
"needs a look at why it fails.", module, node, namesWords(plain, 3)),
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
namesWords(plain, 3)),
Needs: needs,
Actions: actions,
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
}
+205 -131
View File
@@ -63,71 +63,80 @@ var plainWordings = map[string]func(conditions.Observation) words{
// Machines and their node-engine.
"silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " is not answering",
w := words{Headline: m + " is not answering",
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
"normal; if %s should be on, check that it is powered and online.", m, m),
"normal.", m),
Resolved: m + " answers again"}
if o.Severity == conditions.Urgent {
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
"nothing changes anywhere until it is back.", m)
}
return w
}),
"sent-not-reported": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " has not confirmed a change",
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them. It "+
"usually catches up by itself; if it stays, %s may be stuck.", m, m, m),
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
"It usually catches up by itself.", m, m),
Resolved: m + " applied the change"}
}),
"tools-silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Tools on " + m + " are not answering",
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
"the mesh. A machine asleep is normal.", m, m),
"the mesh until it does. A machine asleep is normal.", m, m),
Resolved: "Tools on " + m + " answer again"}
}),
kindAwaitingPush: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " waits for new instructions",
Explanation: fmt.Sprintf("%s's instructions only work once the next push is made, and none has been made "+
"for a while. Nothing is broken; an agent can push to %s.", conditions.Capital(m), m),
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
conditions.Capital(m)),
Resolved: m + " got its new instructions"}
}),
"declaration-refused": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeMesh {
return words{Headline: "No machine can get new instructions",
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
"made. Machines keep running what they have. It needs a look now.",
"made. Machines keep running what they have.",
Resolved: "Machines can get new instructions again"}
}
m := machineOr(o, "a machine")
return words{Headline: m + " cannot get new instructions",
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new reaches "+
"%s. It keeps running what it has. It needs a fix in the catalogue.", m, m),
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
"reaches it. It keeps running what it has.", m),
Resolved: m + " can get new instructions again"}
}),
"own-address-banned": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " has banned the mesh",
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
"mesh is locked out of %s. It needs the ban lifted.", m, m),
"mesh is locked out of it.", m),
Resolved: m + " no longer bans the mesh"}
}),
"core-behind": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " runs an old node-engine",
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds, and no "+
"delivery is bringing it up to date. It works, but misses the newest fixes.", conditions.Capital(m)),
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
Resolved: m + " runs the current node-engine"}
}),
kindMachineUnits: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Failed services on " + m,
Explanation: fmt.Sprintf("Services on %s that no module manages have failed. They are %s's own, so the "+
"mesh does not repair them: mend or remove them there.", m, m),
Resolved: "No failed services on " + m + " any more"}
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
Explanation: "No module manages them, so the mesh does not repair them.",
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}),
kindMachineNetwork: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " has a network problem",
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
"route. Other machines may not reach it, or it them.", conditions.Capital(m)),
"route. It often passes again once a network change settles.", conditions.Capital(m)),
Resolved: m + "'s network is fine again"}
}),
kindNetworkRewritten: worded(func(o conditions.Observation) words {
@@ -141,38 +150,38 @@ var plainWordings = map[string]func(conditions.Observation) words{
m := machineOr(o, "a machine")
return words{Headline: m + " cannot be reached",
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
"asleep is normal; otherwise its tunnel needs a look.", m),
"asleep is normal.", m),
Resolved: m + " can be reached again"}
}),
kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m,
Needs: "decide where its data lives; the details name both places.",
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
"kept the old one because switching would leave the data behind. It needs you to decide.", m),
"kept the old one because switching would leave the data behind.", m),
Resolved: "Resolved: the data source on " + m + " is decided"}
}),
kindBindingMoving: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source moves on " + m,
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider, and its data is "+
"still with the old one. Move the data first.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"}
Needs: "move its data to the new provider before the next push.",
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"}
}),
kindBindingMoved: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source moved on " + m,
Explanation: fmt.Sprintf("A module on %s switched to another provider of its data, and its data may "+
"still be with the old one. Check it found its data.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"}
Needs: "check that it found its data.",
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"}
}),
// Modules and providers.
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: conditions.Capital(thing) + " is not working",
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again; if it stays, "+
"look at why it fails.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " works again"}
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " works again"}
}),
kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
@@ -181,29 +190,29 @@ var plainWordings = map[string]func(conditions.Observation) words{
}
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
"it up. %s may not work until it does.", conditions.Capital(thing), consumer, conditions.Capital(consumer)),
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
}),
kindProviderSilent: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: conditions.Capital(thing) + " went quiet",
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
"reporting. Whether it still fails is not known.", conditions.Capital(thing)),
"reporting.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " reports again"}
}),
kindRetireWaiting: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: "Retiring data on " + thing + " waits for you",
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own. It "+
"needs you to approve or reject retiring it.", conditions.Capital(thing)),
Needs: "approve or reject retiring it; the details list what would go.",
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
conditions.Capital(thing)),
Resolved: "Resolved: the retirement on " + thing + " is decided"}
}),
kindRetireRejected: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: "Unused data kept on " + thing,
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for. Nothing to do; "+
"this is a reminder that it is still there.", thing),
Resolved: "Resolved: the unused data on " + thing + " is gone"}
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
Resolved: "Resolved: the unused data on " + thing + " is gone"}
}),
kindCleanupWaiting: worded(func(o conditions.Observation) words {
var where string
@@ -214,9 +223,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
where = what + " on " + machineOr(o, "a machine")
}
return words{Headline: "Retired data waits for cleanup",
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days. It needs you to delete "+
"it or keep it.", where),
Resolved: "Resolved: the retired data is dealt with"}
Needs: "delete it or keep it; nothing is deleted without your word.",
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
Resolved: "Resolved: the retired data is dealt with"}
}),
// Data.
@@ -224,22 +233,23 @@ var plainWordings = map[string]func(conditions.Observation) words{
m := machineOr(o, "a machine")
return words{Headline: "Data on " + m + " could not be measured",
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
"the data there: not that it is safe either.", m),
"the data there. The next measurement may settle it.", m),
Resolved: "Data on " + m + " is measured again"}
}),
kindProtectionMissing: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " is unprotected",
Needs: "check the storage it is on.",
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
"the mesh can see. It needs you to check its storage.", conditions.Capital(what), machineOr(o, "its machine")),
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is protected again"}
}),
kindDataMissing: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " is gone",
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more. If that was not meant, it needs "+
"restoring from a backup.", what, machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is back"}
Needs: "restore it from a backup, or silence this if you removed it.",
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is back", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}),
kindDataShrank: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
@@ -248,30 +258,33 @@ var plainWordings = map[string]func(conditions.Observation) words{
what = "a dataset"
}
return words{Headline: conditions.Capital(what) + " shrank on " + m,
Explanation: "More than half of what it held is gone within a week. If you meant that, silence it; if " +
"not, restore the last good copy from a backup.",
Resolved: "Resolved: the shrinking on " + m + " is explained"}
Needs: "if you meant it, silence this; if not, restore the last good copy from a backup.",
Explanation: "More than half of what it held is gone within a week.",
Resolved: "Resolved: the shrinking on " + m + " is explained", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}),
kindDataQuiet: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " stopped changing",
Needs: "check that what writes it still runs.",
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
"usual: whatever writes it has stopped.", conditions.Capital(what), machineOr(o, "its machine")),
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is written again"}
}),
kindBackupStale: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
m := machineOr(o, "its machine")
return words{Headline: "No recent backup of " + what,
Needs: fmt.Sprintf("check that backups run on %s.", m),
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
"changed since would be lost.", conditions.Capital(what), machineOr(o, "its machine")),
"changed since would be lost.", conditions.Capital(what), m),
Resolved: conditions.Capital(what) + " is backed up again"}
}),
kindArrayDegraded: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Storage on " + m + " is degraded",
Explanation: fmt.Sprintf("The redundant storage on %s lost a disk or is rebuilding. The data on it is "+
"less protected until it is whole again.", m),
Resolved: "Storage on " + m + " is whole again"}
Needs: fmt.Sprintf("check the disks of %s.", m),
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
Resolved: "Storage on " + m + " is whole again"}
}),
kindEmptyReplacement: worded(func(o conditions.Observation) words {
module := idPart(o, 1)
@@ -282,8 +295,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
module = "a module"
}
return words{Headline: conditions.Capital(module) + " runs on empty data",
Needs: "decide where its data lives: move the data, or move it back.",
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
"elsewhere. It needs you: move the data, or move it back.", conditions.Capital(module), machineOr(o, "its machine")),
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
Resolved: conditions.Capital(module) + " has its data again"}
}),
kindDataHeldTwice: worded(func(o conditions.Observation) words {
@@ -292,40 +306,38 @@ var plainWordings = map[string]func(conditions.Observation) words{
consumer = "a module"
}
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one. It needs you to "+
"decide which to keep.", conditions.Capital(consumer)),
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
Needs: "decide which copy to keep.",
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
}),
// Deliveries and walks.
"stalled": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeDelivery {
return words{Headline: "A delivery is held too long",
Explanation: "A delivery has been held past its time limit. " + conditions.ResolverWords(o),
Resolved: "Resolved: the delivery moves again"}
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
Resolved: "Resolved: the delivery moves again"}
}
return words{Headline: "A delivery is stuck halfway",
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost; if it " +
"stays, an agent can look at why.",
Resolved: "Resolved: the delivery moves again"}
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
Resolved: "Resolved: the delivery moves again"}
}),
kindWalkWaiting: worded(func(o conditions.Observation) words {
return words{Headline: "A delivery is waiting to start",
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
"out) has not said it may start. Nothing to do yet; an agent can start it by hand.",
"out) has not let it start yet.",
Resolved: "Resolved: the delivery is no longer waiting"}
}),
"merge-not-acted": worded(func(o conditions.Observation) words {
return words{Headline: "A merge was never picked up",
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
"sent. An agent can hand it over again.",
"sent. The mesh catches up on missed merges by itself.",
Resolved: "Resolved: the merge was picked up"}
}),
"ask-lost": worded(func(o conditions.Observation) words {
return words{Headline: "A build has no result",
Explanation: "A build was asked for and never answered. What it was for is not delivered until it is " +
"built again.",
Resolved: "Resolved: the build has its result"}
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
Resolved: "Resolved: the build has its result"}
}),
kindRolledBack: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
@@ -336,68 +348,73 @@ var plainWordings = map[string]func(conditions.Observation) words{
if o.Scope == conditions.ScopeCore {
return words{Headline: "A core update was put back on " + m,
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
"before was put back. It needs a look now.", module, m),
"before was put back and runs.", module, m),
Resolved: "Resolved: the core on " + m + " is healthy"}
}
return words{Headline: conditions.Capital(module) + " update put back on " + m,
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
"before. The change is not delivered; it needs a fix in its code.", module, m),
"before, which runs. The change is not delivered until it is fixed.", module, m),
Resolved: "Resolved: " + module + " on " + m + " is settled"}
}),
kindRollbackFailed: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Putting back an update failed on " + m,
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before. What runs "+
"there may be broken; it needs you now.", m),
Resolved: "Resolved: " + m + " runs a good build again"}
Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
Resolved: "Resolved: " + m + " runs a good build again"}
}),
"release-held": worded(func(o conditions.Observation) words {
return words{Headline: "Updates wait for your release",
Explanation: "Some module updates wait for a person to release them. They are not delivered until you do.",
Needs: "release them, or leave them held.",
Explanation: "Some module updates wait for a person to release them, and are not delivered until then.",
Resolved: "Resolved: the held updates are released"}
}),
"facts-stale": worded(func(o conditions.Observation) words {
return words{Headline: "Merge checks use outdated facts",
Explanation: "The facts a merge check judges a change against are out of date, so a check may pass or " +
"fail for the wrong reason.",
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
"by themselves.",
Resolved: "Merge checks use current facts again"}
}),
// The controller and the core.
"controller-deaf": worded(func(o conditions.Observation) words {
return words{Headline: "The controller stopped listening",
Needs: "restart the controller if this stays.",
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
"wait. Changes and repairs do not happen until it recovers.",
Resolved: "The controller listens again"}
}),
"self-check-silent": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh's self-check stopped",
Needs: "restart the controller if this stays.",
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
"Problems may go unnoticed until it runs again.",
Resolved: "The self-check runs again"}
}),
"watchdogs-silent": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh's watchdogs stopped",
Explanation: "The watchdogs, which notice when something expected does not happen, have not run. Missed " +
"signals are not being noticed.",
Needs: "restart the controller if this stays.",
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
"missed signals are not noticed.",
Resolved: "The watchdogs run again"}
}),
"lease-lost": worded(func(o conditions.Observation) words {
return words{Headline: "The controller lost its lease",
Needs: "make sure only one controller runs; the details say which machines claim it.",
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
"not renew it, so two could act at once, or none. It needs a look now.",
"not renew it, so two could act at once, or none.",
Resolved: "The controller holds its lease again"}
}),
"lease-split": worded(func(o conditions.Observation) words {
return words{Headline: "Two controllers may be acting",
Explanation: "The lease that lets one controller act at a time does not agree with its record, so two " +
"could act at once. It needs a look now.",
Resolved: "One controller acts again"}
Needs: "make sure only one controller runs; the details say which machines claim it.",
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
Resolved: "One controller acts again"}
}),
"stale-writer": worded(func(o conditions.Observation) words {
return words{Headline: "Outdated instructions are being sent",
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
"were refused. It is harmless if it stops.",
"were refused, which is harmless.",
Resolved: "Resolved: no more outdated instructions"}
}),
"status-slow": worded(func(o conditions.Observation) words {
@@ -408,32 +425,32 @@ var plainWordings = map[string]func(conditions.Observation) words{
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeBus {
return words{Headline: "The bus is not healthy",
Needs: "check the machine the bus runs on; the details say what fails.",
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
"and tool calls may fail until it recovers.",
Resolved: "The bus is healthy again"}
}
m := machineOr(o, "its machine")
return words{Headline: "The mesh's own software fails on " + m,
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy. What it does there may "+
"not happen until it recovers.", m),
Resolved: "The mesh's own software on " + m + " is healthy again"}
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
Resolved: "The mesh's own software on " + m + " is healthy again"}
}),
"call-hung": worded(func(o conditions.Observation) words {
return words{Headline: "A tool call is hanging",
Explanation: "A call to one of the mesh's tools has run far past its time limit. Whatever waits on it is " +
"stuck.",
Resolved: "Resolved: the tool call ended"}
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
Resolved: "Resolved: the tool call ended"}
}),
"probe-failed": worded(func(o conditions.Observation) words {
return words{Headline: "A check of the mesh could not run",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown, which is not " +
"the same as fine.",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
"again every few minutes.",
Resolved: "The check runs again"}
}),
kindHealersBraked: worded(func(o conditions.Observation) words {
return words{Headline: "All automatic repairs stopped",
Needs: "read the details and decide whether the repairs may run again.",
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
"looks like a loop, so all of them stopped. Nothing repairs itself until this is looked at.",
"looks like a loop, so all of them stopped.",
Resolved: "Automatic repairs run again"}
}),
"healer-wanted": worded(func(o conditions.Observation) words {
@@ -442,8 +459,8 @@ var plainWordings = map[string]func(conditions.Observation) words{
cause = "something"
}
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
Explanation: "A person keeps repairing this by hand. A healer (an automatic repair) is wanted for it; " +
"nothing is broken now.",
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
"broken now.",
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
}),
@@ -451,8 +468,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
"resolver-wrong": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Name lookups fail on " + m,
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly. "+
"Machines that rely on it may not find each other.", m),
Needs: fmt.Sprintf("check that %s is on and online.", m),
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
"machines that rely on it may not find each other.", m),
Resolved: "Name lookups work on " + m + " again"}
}),
"holder-silent": worded(func(o conditions.Observation) words {
@@ -468,50 +486,47 @@ var plainWordings = map[string]func(conditions.Observation) words{
}),
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
return words{Headline: "The bus could not be fully set up",
Explanation: "When instructions were sent, the bus's queues could not all be checked, so a module may " +
"not get its messages.",
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
"again with the next send.",
Resolved: "The bus is fully set up again"}
}),
kindBusMaintenance: worded(func(o conditions.Observation) words {
return words{Headline: "The bus is being upgraded",
Explanation: "The mesh's message system is being upgraded; some things pause until it is done. Nothing " +
"to do.",
Resolved: "The bus upgrade is done"}
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
Resolved: "The bus upgrade is done"}
}),
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
return words{Headline: "The bus upgrade failed",
Explanation: "The upgrade of the mesh's message system did not end healthy in time. It needs you: it can " +
"be put back to the version before.",
Resolved: "Resolved: the bus is healthy after its upgrade"}
Needs: "decide whether to put the bus back to the version before; the details say how.",
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
Resolved: "Resolved: the bus is healthy after its upgrade"}
}),
kindConsumerBehind: worded(func(o conditions.Observation) words {
return words{Headline: "Messages pile up for a listener",
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
"healer may restart it.",
"healer restarts it if it stays.",
Resolved: "The listener caught up"}
}),
"consumer-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is set up wrong",
Explanation: "One of the mesh's listeners on the bus is missing or not as the controller defines it, so " +
"some messages may reach nobody.",
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
"it up again.",
Resolved: "The listener is set up right again"}
}),
"consumer-lost": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is gone",
Explanation: "One of the mesh's listeners is no longer on the bus, so what it should receive reaches " +
"nobody.",
Resolved: "The listener is back"}
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
Resolved: "The listener is back"}
}),
"slow-consumer": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is too slow",
Explanation: "The bus reports a listener too slow to keep up, so messages to it are dropped or late.",
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
Resolved: "The listener keeps up again"}
}),
"max-deliveries": worded(func(o conditions.Observation) words {
return words{Headline: "A message could not be handled",
Explanation: "The bus gave up on a message after trying to hand it over too many times. What it carried " +
"did not happen.",
Resolved: "Resolved: messages are handled again"}
Explanation: "The bus gave up on a message after trying to hand it over too many times.",
Resolved: "Resolved: messages are handled again"}
}),
"refused": worded(func(o conditions.Observation) words {
return words{Headline: "The bus refuses some messages",
@@ -520,21 +535,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
}),
"stream-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "Part of the bus's storage is wrong",
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps " +
"there may be lost.",
Resolved: "The bus's storage is right again"}
Needs: "check the machine the bus runs on; the details say what is missing.",
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
Resolved: "The bus's storage is right again"}
}),
"archives-unheld": worded(func(o conditions.Observation) words {
return words{Headline: "Kept builds are about to be cleaned up",
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them. " +
"Nothing to do unless one is still needed.",
Resolved: "Resolved: every kept build is claimed"}
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
Resolved: "Resolved: every kept build is claimed"}
}),
"archives-missing": worded(func(o conditions.Observation) words {
return words{Headline: "Kept builds are missing",
Explanation: "Some builds the mesh relies on are no longer in the store. A machine that needs one cannot " +
"get it until it is built again.",
Resolved: "Resolved: the kept builds are back"}
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
Resolved: "Resolved: the kept builds are back"}
}),
}
@@ -602,18 +615,68 @@ func deliveryName(modules []string, repository string) string {
return repoName(repository) + " delivery"
}
// waitingAct is what the operator does about a walk waiting for its delivery's word.
func waitingAct(severity conditions.Severity) string {
// walkWaitingWords explains a walk waiting for its delivery's word.
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
what := deliveryWhat(w.modules, w.repository)
if severity == conditions.Urgent {
return "Mesh-delivery may be broken. It needs you: have an agent look at mesh-delivery, or start the " +
"delivery by hand."
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
}
return fmt.Sprintf("Nothing to do yet; it becomes urgent after %s. To start it now, have an agent start it by hand.",
humanDuration(waitUrgentAfter))
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
what, humanDuration(in), humanDuration(waitUrgentAfter))
}
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
func waitingNeeds(severity conditions.Severity) string {
if severity == conditions.Urgent {
return "start it, or stop it."
}
return ""
}
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
// the module that should have said go is the one not answering.
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
if severity != conditions.Urgent {
return nil
}
return []conditions.Action{
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
}
}
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
var actions []conditions.Action
for _, r := range rs {
if strings.Contains(r.Reason, "relogin needed") {
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
}
if r.Kind == link.KindUnit && len(actions) < 2 {
scope := "system"
if strings.Contains(r.Reason, "account's own") {
scope = "user"
}
label := "Restart"
if len(actions) > 0 {
label = "Restart " + unitPlainWords(r.Target)
}
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
}
}
if len(actions) > 0 {
return "restart it; if it fails again, the details say why.", actions
}
return "", nil
}
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
func stalledWords(l stalledLine, o conditions.Observation) (string, string, string) {
func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
actions []conditions.Action) {
repository, _, _ := strings.Cut(l.ID, "@")
name := repoName(repository)
held := l.State
@@ -624,13 +687,24 @@ func stalledWords(l stalledLine, o conditions.Observation) (string, string, stri
if d, err := time.ParseDuration(l.For); err == nil {
long = "for " + humanDuration(d)
}
act := conditions.ResolverWords(o)
if o.Resolver == conditions.ResolverOperator {
act = "It waits for you: release it, or stop it."
switch held {
case "held":
needs = "release it, or stop it."
actions = []conditions.Action{
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
}
case "ready", "checked":
needs = "merge its pull request, or close it."
default:
needs = "stop it, or read the details to see what it waits for."
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
}
}
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
fmt.Sprintf("A delivery of %s has been %s %s, past its limit. %s", name, held, long, act),
fmt.Sprintf("Delivery of %s is no longer %s", name, held)
fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
}
// causeWords is a hand-act's cause as a person says it.
+74 -30
View File
@@ -14,29 +14,40 @@ import (
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
// plainExample checks one finding's words: plain, within their bounds, and saying what the operator needs.
func plainExample(t *testing.T, o conditions.Observation, headline string, says ...string) {
// plainExample checks one finding's words as the keeper will keep them: plain, within their bounds,
// opened by the verdict, and saying what the operator needs; and its actions by label.
func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) {
t.Helper()
machines := append([]string{o.Machine}, o.Also...)
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
Actions: o.Actions}
if why, ok := conditions.PlainWords(w, machines...); !ok {
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
}
if o.Headline != headline {
t.Errorf("%s: headline %q, want %q", o.Key(), o.Headline, headline)
said := conditions.Verdict(o.Needs, o.Explanation)
if o.Headline != headline || said != explanation {
t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation)
}
for _, s := range says {
if !strings.Contains(o.Explanation+" | "+o.Resolved, s) {
t.Errorf("%s: the words do not say %q:\n %s | %s", o.Key(), s, o.Explanation, o.Resolved)
}
var labels []string
for _, a := range o.Actions {
labels = append(labels, a.Label)
}
t.Logf("\nBEFORE: %s\nAFTER: %s\n %s\nCLEARS: %s", o.Summary, o.Headline, o.Explanation, o.Resolved)
if strings.Join(labels, ",") != strings.Join(actions, ",") {
t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions)
}
quiet := ""
if o.Needs == "" && o.Severity == conditions.Warning {
quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)"
}
t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet,
said, labels, o.Resolved)
}
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
// `plans go plan-1791454185265004861 --why …` starts it by hand".
func TestADeliveryWaitingIsSaidByItsModulesAndWhatToDo(t *testing.T) {
// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no
// popup; past them, the operator starts or stops it from the notification.
func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) {
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
f := calm(now)
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
@@ -45,36 +56,59 @@ func TestADeliveryWaitingIsSaidByItsModulesAndWhatToDo(t *testing.T) {
if len(got) != 1 {
t.Fatalf("%+v", got)
}
plainExample(t, got[0], "openrazer delivery waiting to start", "56 minutes", "mesh-delivery (the module that decides",
"Nothing to do yet", "openrazer delivery no longer waiting")
plainExample(t, got[0], "openrazer delivery waiting to start",
"Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+
"for you if it still waits after 4 hours.")
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
}
// Past four hours it is urgent, and says it needs the operator.
// Past four hours it is urgent, and offers the controller's own answers.
f.waits[0].since = now.Add(-5 * time.Hour)
got = watchWaits(f)
plainExample(t, got[0], "openrazer delivery waiting to start", "It needs you")
plainExample(t, got[0], "openrazer delivery waiting to start",
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
"be stuck.", "Start", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
t.Errorf("start: %+v", a)
}
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
t.Errorf("stop: %+v", a)
}
// Many modules are counted, not listed in the headline.
f.waits[0].modules = []string{"a", "b", "c", "d"}
got = watchWaits(f)
plainExample(t, got[0], "mesh-catalog delivery waiting to start", "4 modules of mesh-catalog")
if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") {
t.Errorf("%+v", got[0])
}
}
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
// account's own service manager (exit-code)".
func TestAModuleUnhealthyNamesItsServiceInWords(t *testing.T) {
// account's own service manager (exit-code)". The operator restarts it from the notification.
func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
plainExample(t, o, "openrazer not working on g14", "is not healthy: its service openrazer-daemon stopped with an error",
"openrazer works again on g14")
plainExample(t, o, "openrazer not working on g14",
"Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
t.Errorf("restart: %+v", a)
}
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
}
}
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). Each is the
// machine's own: mend or remove it there, or have a module place it".
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …"
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
@@ -83,12 +117,14 @@ func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
t.Fatal("not raised")
}
plainExample(t, o, "3 failed services on shanks",
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed", "mend or remove them on shanks")
"Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+
"mesh does not repair them.", "Silence for a week")
}
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools,
// through the mesh-controller seat: a healer is wanted for it".
func TestAHealerWantedSaysNothingIsBroken(t *testing.T) {
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through
// the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup.
func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
f := calm(now)
for i := 0; i < 35; i++ {
@@ -100,15 +136,22 @@ func TestAHealerWantedSaysNothingIsBroken(t *testing.T) {
if len(got) != 1 {
t.Fatalf("%+v", got)
}
plainExample(t, got[0], "Push keeps being fixed by hand", "35 times in 14 days", "nothing is broken now")
plainExample(t, got[0], "Push keeps being fixed by hand",
"Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+
"wanted for it. Nothing is broken now.")
}
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
func TestADeliveryHeldSaysItWaitsForTheOperator(t *testing.T) {
func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
plainExample(t, got[0], "Delivery of hq held for 36 hours", "It waits for you: release it, or stop it.")
plainExample(t, got[0], "Delivery of hq held for 36 hours",
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
"Release", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
t.Errorf("release: %+v", a)
}
}
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
@@ -134,6 +177,7 @@ func TestEveryWordingIsPlain(t *testing.T) {
for _, s := range subjects {
s.Kind = kind
w := fn(s)
w.Explanation = conditions.Verdict(w.Needs, w.Explanation)
if why, ok := conditions.PlainWords(w, s.Machine); !ok {
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
}
+9 -9
View File
@@ -324,7 +324,7 @@ func watchPlans(f *signalFacts) []conditions.Observation {
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway",
Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+
"usual. Nothing is lost; if it stays, an agent can look at why.", humanDuration(f.now.Sub(p.entered))),
"usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))),
Resolved: "Delivery of " + repoName(p.repository) + " moves again",
Severity: conditions.Warning,
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
@@ -357,12 +357,12 @@ func watchWaits(f *signalFacts) []conditions.Observation {
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the "+
"module that decides when a delivery goes out) to let it start. %s", deliveryWhat(w.modules, w.repository),
humanDuration(in), waitingAct(severity)),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: walkWaitingWords(w, in, severity),
Needs: waitingNeeds(severity),
Actions: waitingActions(w, severity),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
}
return out
}
@@ -692,8 +692,8 @@ func watchHandActs(f *signalFacts) []conditions.Observation {
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
Said: strings.Join(acts, "; "),
Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand",
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days. A healer (an automatic repair) "+
"is wanted for it; nothing is broken now.", causeWords(cause), repeated[cause],
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is "+
"wanted for it. Nothing is broken now.", causeWords(cause), repeated[cause],
int(handActsWithin.Hours()/24)),
Resolved: "Resolved: no more hand repairs of " + causeWords(cause)})
}
+6
View File
@@ -134,6 +134,10 @@ type Condition struct {
Headline string `json:"headline"`
Explanation string `json:"explanation"`
Resolved string `json:"resolved"`
// Needs is what the operator does about it, one sentence; empty when nothing (the explanation then
// opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go).
Needs string `json:"needs"`
Actions []Action `json:"actions"`
// Evidence is the newest observations, at most KeptEvidence, newest first.
Evidence []Evidence `json:"evidence"`
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
@@ -187,6 +191,8 @@ type Observation struct {
Headline string
Explanation string
Resolved string
Needs string
Actions []Action
// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
// here, never in Summary**: an address, a socket's error, a path or a name with its domain is
// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
+91 -23
View File
@@ -11,7 +11,12 @@ package conditions
// g14"), the title of every message about it;
// - an **explanation**: one or two plain sentences — what happened, what it means for the operator,
// and whether they need to do anything;
// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14").
// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14");
// - a **verdict**, which every explanation opens with: "Nothing for you to do." or "Needs you:" and one
// concrete thing the operator can do themselves (needs). Never "have an agent …": the operator is not
// asked to open a session to understand or answer a notification;
// - **actions**: what the operator may answer from the notification itself (Release, Stop, Restart,
// Silence), each the seat verb the operator channel calls when it is chosen, naming the operator.
//
// They are made here, where the condition is made, so every channel gets them — the desktop today,
// others later — and none has to guess what a key means. A producer may say them itself; otherwise the
@@ -29,17 +34,45 @@ import (
"github.com/novox/mesh-controller/internal/outward"
)
// Words are what the operator reads of a condition.
// Words are what the operator reads of a condition. Explanation is said after the verdict, which
// plainly writes from Needs: empty is "Nothing for you to do.", else "Needs you: " and Needs.
type Words struct {
Headline string
Explanation string
Resolved string
Needs string
Actions []Action
}
// Action is one answer the operator may give from a notification: a label, and the seat verb the
// operator channel calls with these arguments (and a why naming the operator and the label) when it is
// chosen. Machine is set for a seat every machine holds. An argument "why" given empty is the operator
// channel's to fill: it names the operator, the channel and the label chosen.
type Action struct {
Label string `json:"label"`
Verb string `json:"verb"`
Machine string `json:"machine,omitempty"`
Arguments map[string]string `json:"arguments,omitempty"`
}
// The two verdicts an explanation opens with.
const (
NothingToDo = "Nothing for you to do."
NeedsYou = "Needs you:"
)
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
// answer to a condition the operator decided to live with.
func SilenceAction(key string) Action {
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
}
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
const (
HeadlineMax = 60
ExplanationMax = 360
ExplanationMax = 420
NeedsMax = 120
)
var (
@@ -75,6 +108,7 @@ var (
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
)
@@ -124,6 +158,27 @@ func PlainWords(w Words, machines ...string) (string, bool) {
if len(w.Explanation) > ExplanationMax {
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
}
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
return "explanation: sends the operator elsewhere (" + m + ")", false
}
}
if w.Needs != "" {
if why, ok := Plain(w.Needs, machines...); !ok {
return "needs: " + why, false
}
if m := agentWord.FindString(w.Needs); m != "" {
return "needs: not something the operator does themselves (" + m + ")", false
}
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
return fmt.Sprintf("needs: one sentence of at most %d characters, ending in a full stop", NeedsMax), false
}
}
for _, a := range w.Actions {
if a.Label == "" || len(a.Label) > 24 || a.Verb == "" {
return fmt.Sprintf("action %q: a short label and a verb", a.Label), false
}
}
if why, ok := Plain(w.Resolved, machines...); !ok {
return "resolved: " + why, false
}
@@ -136,7 +191,8 @@ func PlainWords(w Words, machines ...string) (string, bool) {
// plainly gives an observation its plain words: its own, its kind's, or its scope's.
func plainly(o Observation) Observation {
machines := append([]string{o.Machine}, o.Also...)
given := Words{o.Headline, o.Explanation, o.Resolved}
given := Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
Actions: o.Actions}
var w Words
from := ""
switch {
@@ -154,6 +210,12 @@ func plainly(o Observation) Observation {
if given.Resolved != "" {
w.Resolved = given.Resolved
}
if given.Needs != "" {
w.Needs = given.Needs
}
if len(given.Actions) > 0 {
w.Actions = given.Actions
}
}
}
if w.Headline != "" && w.Resolved == "" {
@@ -170,17 +232,27 @@ func plainly(o Observation) Observation {
}
w = scopeWords(o)
}
o.Headline, o.Explanation, o.Resolved = w.Headline, w.Explanation, w.Resolved
o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
o.Explanation = Verdict(w.Needs, w.Explanation)
return o
}
// Escalated is the sentence a condition's explanation ends with once a healer gave up on it.
const Escalated = "The mesh tried to repair it and could not: it needs you now."
// Verdict is an explanation opened by its verdict.
func Verdict(needs, explanation string) string {
if needs == "" {
return strings.TrimSpace(NothingToDo + " " + explanation)
}
return strings.TrimSpace(NeedsYou + " " + needs + " " + explanation)
}
// escalatedWords says in the explanation that a healer gave up, whatever the words said before.
// Escalated is what a condition needs once a healer gave up on it, where its words needed nothing.
const Escalated = "the mesh tried to repair this and could not; read the details to decide what to do."
// escalatedWords turns "Nothing for you to do." into "Needs you:" once a healer gave up.
func escalatedWords(c *Condition) {
if c.Escalated() && !strings.HasSuffix(c.Explanation, Escalated) {
c.Explanation = strings.TrimSpace(c.Explanation + " " + Escalated)
if c.Escalated() && c.Needs == "" {
c.Needs = Escalated
c.Explanation = Verdict(c.Needs, strings.TrimSpace(strings.TrimPrefix(c.Explanation, NothingToDo)))
}
}
@@ -191,13 +263,14 @@ func scopeWords(o Observation) Words {
what := strings.ReplaceAll(o.Kind, "-", " ")
w := Words{
Headline: Capital(thing) + " needs a look",
Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s. %s", what, thing, ResolverWords(o)),
Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s.", what, thing),
Resolved: "Resolved: " + thing + " is fine again",
Needs: ResolverNeeds(o),
}
if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok {
w = Words{Headline: "Something in the mesh needs a look",
Explanation: "The mesh noticed a problem it has no plain words for yet. " + ResolverWords(o),
Resolved: "Resolved: the mesh is fine again"}
Explanation: "The mesh noticed a problem it has no plain words for yet.",
Resolved: "Resolved: the mesh is fine again", Needs: ResolverNeeds(o)}
}
return w
}
@@ -244,17 +317,12 @@ func ThingWords(o Observation) string {
return "the mesh"
}
// ResolverWords says whether the operator needs to act, from who resolves it.
func ResolverWords(o Observation) string {
switch {
case o.Resolver == ResolverOperator:
return "It needs you: the mesh does not repair this by itself."
case strings.HasPrefix(o.Resolver, "healer:"):
return "A healer is working on it; nothing to do unless it stays."
case o.Severity == Urgent:
return "It clears by itself once it is fixed, but needs a look now."
// ResolverNeeds is what the operator needs to do, from who resolves it: nothing, unless only a person can.
func ResolverNeeds(o Observation) string {
if o.Resolver == ResolverOperator {
return "the mesh does not repair this by itself; read the details to decide what to do."
}
return "Nothing to do yet: it clears by itself once it is fixed."
return ""
}
// Capital is s with its first letter upper case.
+32 -6
View File
@@ -79,7 +79,7 @@ func TestAConditionCarriesItsPlainWords(t *testing.T) {
}
said := settled(t, told, 1)
body, _ := json.Marshal(said[0])
for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"The mesh has not heard from it."`,
for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"Nothing for you to do. The mesh has not heard from it."`,
`"resolved":"ace answers again"`} {
if !strings.Contains(string(body), field) {
t.Errorf("the event does not carry %s: %s", field, body)
@@ -100,7 +100,7 @@ func TestAConditionCarriesItsPlainWords(t *testing.T) {
// Words that are not plain, and a kind with none, are said from the scope — and reported.
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded",
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"})
if c.Headline != "Openrazer on g14 needs a look" || !strings.Contains(c.Explanation, "Nothing to do yet") {
if c.Headline != "Openrazer on g14 needs a look" || !strings.HasPrefix(c.Explanation, NothingToDo) {
t.Errorf("the scope's words: %+v", c)
}
c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent",
@@ -113,13 +113,39 @@ func TestAConditionCarriesItsPlainWords(t *testing.T) {
}
}
// **An explanation opens with its verdict, and never sends the operator to an agent.**
func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
if got := Verdict("", "It may be asleep."); got != "Nothing for you to do. It may be asleep." {
t.Errorf("%q", got)
}
if got := Verdict("release it, or stop it.", "It is held."); got != "Needs you: release it, or stop it. It is held." {
t.Errorf("%q", got)
}
for _, w := range []Words{
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
} {
if _, ok := PlainWords(w); ok {
t.Errorf("passed: %+v", w)
}
}
if why, ok := PlainWords(Words{Headline: "push keeps being fixed by hand", Explanation: "A person repaired " +
"push by hand 35 times.", Resolved: "Resolved", Needs: "release it, or stop it.",
Actions: []Action{{Label: "Release", Verb: "mesh-delivery.release"}}}); !ok {
t.Errorf("refused: %s", why)
}
}
// **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the
// budget is spent, the operator is needed.
func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
k, _, _, _ := keeper(t)
ctx := t.Context()
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "Nothing to do yet."}
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
if _, err := k.Observe(ctx, o); err != nil {
t.Fatal(err)
}
@@ -127,11 +153,11 @@ func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(c.Explanation, Escalated) {
t.Fatalf("escalated: %q", c.Explanation)
if want := "Needs you: " + Escalated + " It may be asleep."; c.Explanation != want || c.Needs != Escalated {
t.Fatalf("escalated: %q, want %q", c.Explanation, want)
}
c, _ = k.Observe(ctx, o)
if strings.Count(c.Explanation, Escalated) != 1 {
if strings.Count(c.Explanation, Escalated) != 1 || strings.Contains(c.Explanation, NothingToDo) {
t.Errorf("seen again after escalation: %q", c.Explanation)
}
}
+2 -2
View File
@@ -190,7 +190,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if !found {
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved, Evidence: []Evidence{{At: now, Said: said}},
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}},
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
Resolver: orSelf(o.Resolver)}
change := ChangeRaised
@@ -240,7 +240,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
// The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
c.Headline, c.Explanation, c.Resolved = o.Headline, o.Explanation, o.Resolved
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
escalatedWords(&c)
if o.Machine != "" {
c.Subject.Machine = o.Machine