Open every explanation with what the operator needs to do, and offer the answers
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/plain-notifications merged: every member is merged

The operator could not tell from a notification whether to act, and was told to
have an agent do it. Each condition now says "Nothing for you to do." or
"Needs you:" with one thing they can do themselves, and carries the actions the
operator channel performs when chosen (hq ADR 0253).
This commit is contained in:
jochen
2026-10-08 13:58:53 +02:00
parent 7df72edd2b
commit 1a4305213d
10 changed files with 430 additions and 207 deletions
+1 -1
View File
@@ -113,7 +113,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
if l.operatorsOnly() { if l.operatorsOnly() {
o.Resolver = conditions.ResolverOperator o.Resolver = conditions.ResolverOperator
} }
o.Headline, o.Explanation, o.Resolved = stalledWords(l, o) o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
out = append(out, o) out = append(out, o)
} }
return out return out
+4 -2
View File
@@ -117,8 +117,10 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
failed = fmt.Sprintf("%d failed services", len(u.Failed)) failed = fmt.Sprintf("%d failed services", len(u.Failed))
} }
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node) o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them: mend or "+ o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
"remove them on %s.", node, namesWords(plain, 3), node) o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
node, namesWords(plain, 3))
o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
o.Resolved = "No failed services on " + node + " any more" o.Resolved = "No failed services on " + node + " any more"
return o, true return o, true
} }
+6 -3
View File
@@ -167,7 +167,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
o.Severity = conditions.Urgent o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters) o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it, so this is urgent.", len(waiters)) o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
} }
} }
seen[o.Key()] = true seen[o.Key()] = true
@@ -243,6 +243,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence. // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
var words, said, plain []string var words, said, plain []string
needs, actions := moduleNeeds(node, rs)
for _, r := range rs { for _, r := range rs {
plain = append(plain, resourcePlainWords(r)) plain = append(plain, resourcePlainWords(r))
if r.Kind == link.KindUnit { if r.Kind == link.KindUnit {
@@ -263,8 +264,10 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")), Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
Said: strings.Join(said, "; "), Said: strings.Join(said, "; "),
Headline: fmt.Sprintf("%s not working on %s", module, node), Headline: fmt.Sprintf("%s not working on %s", module, node),
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again; if it stays, it "+ Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
"needs a look at why it fails.", module, node, namesWords(plain, 3)), namesWords(plain, 3)),
Needs: needs,
Actions: actions,
Resolved: fmt.Sprintf("%s works again on %s", module, node)} Resolved: fmt.Sprintf("%s works again on %s", module, node)}
} }
+205 -131
View File
@@ -63,71 +63,80 @@ var plainWordings = map[string]func(conditions.Observation) words{
// Machines and their node-engine. // Machines and their node-engine.
"silent": worded(func(o conditions.Observation) words { "silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " is not answering", w := words{Headline: m + " is not answering",
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+ Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
"normal; if %s should be on, check that it is powered and online.", m, m), "normal.", m),
Resolved: m + " answers again"} Resolved: m + " answers again"}
if o.Severity == conditions.Urgent {
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
"nothing changes anywhere until it is back.", m)
}
return w
}), }),
"sent-not-reported": worded(func(o conditions.Observation) words { "sent-not-reported": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " has not confirmed a change", return words{Headline: m + " has not confirmed a change",
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them. It "+ Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
"usually catches up by itself; if it stays, %s may be stuck.", m, m, m), "It usually catches up by itself.", m, m),
Resolved: m + " applied the change"} Resolved: m + " applied the change"}
}), }),
"tools-silent": worded(func(o conditions.Observation) words { "tools-silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "Tools on " + m + " are not answering", return words{Headline: "Tools on " + m + " are not answering",
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+ Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
"the mesh. A machine asleep is normal.", m, m), "the mesh until it does. A machine asleep is normal.", m, m),
Resolved: "Tools on " + m + " answer again"} Resolved: "Tools on " + m + " answer again"}
}), }),
kindAwaitingPush: worded(func(o conditions.Observation) words { kindAwaitingPush: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " waits for new instructions", return words{Headline: m + " waits for new instructions",
Explanation: fmt.Sprintf("%s's instructions only work once the next push is made, and none has been made "+ Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
"for a while. Nothing is broken; an agent can push to %s.", conditions.Capital(m), m), conditions.Capital(m)),
Resolved: m + " got its new instructions"} Resolved: m + " got its new instructions"}
}), }),
"declaration-refused": worded(func(o conditions.Observation) words { "declaration-refused": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeMesh { if o.Scope == conditions.ScopeMesh {
return words{Headline: "No machine can get new instructions", return words{Headline: "No machine can get new instructions",
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " + Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
"made. Machines keep running what they have. It needs a look now.", "made. Machines keep running what they have.",
Resolved: "Machines can get new instructions again"} Resolved: "Machines can get new instructions again"}
} }
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " cannot get new instructions", return words{Headline: m + " cannot get new instructions",
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new reaches "+ Needs: "decide whether to undo the last catalogue change; the details say what fails.",
"%s. It keeps running what it has. It needs a fix in the catalogue.", m, m), Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
"reaches it. It keeps running what it has.", m),
Resolved: m + " can get new instructions again"} Resolved: m + " can get new instructions again"}
}), }),
"own-address-banned": worded(func(o conditions.Observation) words { "own-address-banned": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " has banned the mesh", return words{Headline: m + " has banned the mesh",
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+ Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
"mesh is locked out of %s. It needs the ban lifted.", m, m), "mesh is locked out of it.", m),
Resolved: m + " no longer bans the mesh"} Resolved: m + " no longer bans the mesh"}
}), }),
"core-behind": worded(func(o conditions.Observation) words { "core-behind": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " runs an old node-engine", return words{Headline: m + " runs an old node-engine",
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds, and no "+ Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
"delivery is bringing it up to date. It works, but misses the newest fixes.", conditions.Capital(m)), "works, and gets the newer one with a later delivery.", conditions.Capital(m)),
Resolved: m + " runs the current node-engine"} Resolved: m + " runs the current node-engine"}
}), }),
kindMachineUnits: worded(func(o conditions.Observation) words { kindMachineUnits: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "Failed services on " + m, return words{Headline: "Failed services on " + m,
Explanation: fmt.Sprintf("Services on %s that no module manages have failed. They are %s's own, so the "+ Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
"mesh does not repair them: mend or remove them there.", m, m), Explanation: "No module manages them, so the mesh does not repair them.",
Resolved: "No failed services on " + m + " any more"} Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}), }),
kindMachineNetwork: worded(func(o conditions.Observation) words { kindMachineNetwork: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " has a network problem", return words{Headline: m + " has a network problem",
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+ Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
"route. Other machines may not reach it, or it them.", conditions.Capital(m)), "route. It often passes again once a network change settles.", conditions.Capital(m)),
Resolved: m + "'s network is fine again"} Resolved: m + "'s network is fine again"}
}), }),
kindNetworkRewritten: worded(func(o conditions.Observation) words { kindNetworkRewritten: worded(func(o conditions.Observation) words {
@@ -141,38 +150,38 @@ var plainWordings = map[string]func(conditions.Observation) words{
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: m + " cannot be reached", return words{Headline: m + " cannot be reached",
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+ Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
"asleep is normal; otherwise its tunnel needs a look.", m), "asleep is normal.", m),
Resolved: m + " can be reached again"} Resolved: m + " can be reached again"}
}), }),
kindBindingKept: worded(func(o conditions.Observation) words { kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m, return words{Headline: "A module's data source is held on " + m,
Needs: "decide where its data lives; the details name both places.",
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+ Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
"kept the old one because switching would leave the data behind. It needs you to decide.", m), "kept the old one because switching would leave the data behind.", m),
Resolved: "Resolved: the data source on " + m + " is decided"} Resolved: "Resolved: the data source on " + m + " is decided"}
}), }),
kindBindingMoving: worded(func(o conditions.Observation) words { kindBindingMoving: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "A module's data source moves on " + m, return words{Headline: "A module's data source moves on " + m,
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider, and its data is "+ Needs: "move its data to the new provider before the next push.",
"still with the old one. Move the data first.", m), Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"} Resolved: "Resolved: the data source move on " + m + " is settled"}
}), }),
kindBindingMoved: worded(func(o conditions.Observation) words { kindBindingMoved: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "A module's data source moved on " + m, return words{Headline: "A module's data source moved on " + m,
Explanation: fmt.Sprintf("A module on %s switched to another provider of its data, and its data may "+ Needs: "check that it found its data.",
"still be with the old one. Check it found its data.", m), Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"} Resolved: "Resolved: the data source move on " + m + " is settled"}
}), }),
// Modules and providers. // Modules and providers.
kindModuleUnhealthy: worded(func(o conditions.Observation) words { kindModuleUnhealthy: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o) thing := conditions.ThingWords(o)
return words{Headline: conditions.Capital(thing) + " is not working", return words{Headline: conditions.Capital(thing) + " is not working",
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again; if it stays, "+ Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
"look at why it fails.", conditions.Capital(thing)), Resolved: conditions.Capital(thing) + " works again"}
Resolved: conditions.Capital(thing) + " works again"}
}), }),
kindProviderFailing: worded(func(o conditions.Observation) words { kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2) thing, consumer := conditions.ThingWords(o), idPart(o, 2)
@@ -181,29 +190,29 @@ var plainWordings = map[string]func(conditions.Observation) words{
} }
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer, return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+ Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
"it up. %s may not work until it does.", conditions.Capital(thing), consumer, conditions.Capital(consumer)), "it up. It tries again by itself.", conditions.Capital(thing), consumer),
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"} Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
}), }),
kindProviderSilent: worded(func(o conditions.Observation) words { kindProviderSilent: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o) thing := conditions.ThingWords(o)
return words{Headline: conditions.Capital(thing) + " went quiet", return words{Headline: conditions.Capital(thing) + " went quiet",
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+ Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
"reporting. Whether it still fails is not known.", conditions.Capital(thing)), "reporting.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " reports again"} Resolved: conditions.Capital(thing) + " reports again"}
}), }),
kindRetireWaiting: worded(func(o conditions.Observation) words { kindRetireWaiting: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o) thing := conditions.ThingWords(o)
return words{Headline: "Retiring data on " + thing + " waits for you", return words{Headline: "Retiring data on " + thing + " waits for you",
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own. It "+ Needs: "approve or reject retiring it; the details list what would go.",
"needs you to approve or reject retiring it.", conditions.Capital(thing)), Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
conditions.Capital(thing)),
Resolved: "Resolved: the retirement on " + thing + " is decided"} Resolved: "Resolved: the retirement on " + thing + " is decided"}
}), }),
kindRetireRejected: worded(func(o conditions.Observation) words { kindRetireRejected: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o) thing := conditions.ThingWords(o)
return words{Headline: "Unused data kept on " + thing, return words{Headline: "Unused data kept on " + thing,
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for. Nothing to do; "+ Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
"this is a reminder that it is still there.", thing), Resolved: "Resolved: the unused data on " + thing + " is gone"}
Resolved: "Resolved: the unused data on " + thing + " is gone"}
}), }),
kindCleanupWaiting: worded(func(o conditions.Observation) words { kindCleanupWaiting: worded(func(o conditions.Observation) words {
var where string var where string
@@ -214,9 +223,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
where = what + " on " + machineOr(o, "a machine") where = what + " on " + machineOr(o, "a machine")
} }
return words{Headline: "Retired data waits for cleanup", return words{Headline: "Retired data waits for cleanup",
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days. It needs you to delete "+ Needs: "delete it or keep it; nothing is deleted without your word.",
"it or keep it.", where), Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
Resolved: "Resolved: the retired data is dealt with"} Resolved: "Resolved: the retired data is dealt with"}
}), }),
// Data. // Data.
@@ -224,22 +233,23 @@ var plainWordings = map[string]func(conditions.Observation) words{
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "Data on " + m + " could not be measured", return words{Headline: "Data on " + m + " could not be measured",
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+ Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
"the data there: not that it is safe either.", m), "the data there. The next measurement may settle it.", m),
Resolved: "Data on " + m + " is measured again"} Resolved: "Data on " + m + " is measured again"}
}), }),
kindProtectionMissing: worded(func(o conditions.Observation) words { kindProtectionMissing: worded(func(o conditions.Observation) words {
what, _ := dataWords(o) what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " is unprotected", return words{Headline: conditions.Capital(what) + " is unprotected",
Needs: "check the storage it is on.",
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+ Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
"the mesh can see. It needs you to check its storage.", conditions.Capital(what), machineOr(o, "its machine")), "the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is protected again"} Resolved: conditions.Capital(what) + " is protected again"}
}), }),
kindDataMissing: worded(func(o conditions.Observation) words { kindDataMissing: worded(func(o conditions.Observation) words {
what, _ := dataWords(o) what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " is gone", return words{Headline: conditions.Capital(what) + " is gone",
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more. If that was not meant, it needs "+ Needs: "restore it from a backup, or silence this if you removed it.",
"restoring from a backup.", what, machineOr(o, "its machine")), Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is back"} Resolved: conditions.Capital(what) + " is back", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}), }),
kindDataShrank: worded(func(o conditions.Observation) words { kindDataShrank: worded(func(o conditions.Observation) words {
what, _ := dataWords(o) what, _ := dataWords(o)
@@ -248,30 +258,33 @@ var plainWordings = map[string]func(conditions.Observation) words{
what = "a dataset" what = "a dataset"
} }
return words{Headline: conditions.Capital(what) + " shrank on " + m, return words{Headline: conditions.Capital(what) + " shrank on " + m,
Explanation: "More than half of what it held is gone within a week. If you meant that, silence it; if " + Needs: "if you meant it, silence this; if not, restore the last good copy from a backup.",
"not, restore the last good copy from a backup.", Explanation: "More than half of what it held is gone within a week.",
Resolved: "Resolved: the shrinking on " + m + " is explained"} Resolved: "Resolved: the shrinking on " + m + " is explained", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}), }),
kindDataQuiet: worded(func(o conditions.Observation) words { kindDataQuiet: worded(func(o conditions.Observation) words {
what, _ := dataWords(o) what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " stopped changing", return words{Headline: conditions.Capital(what) + " stopped changing",
Needs: "check that what writes it still runs.",
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+ Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
"usual: whatever writes it has stopped.", conditions.Capital(what), machineOr(o, "its machine")), "usual.", conditions.Capital(what), machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is written again"} Resolved: conditions.Capital(what) + " is written again"}
}), }),
kindBackupStale: worded(func(o conditions.Observation) words { kindBackupStale: worded(func(o conditions.Observation) words {
what, _ := dataWords(o) what, _ := dataWords(o)
m := machineOr(o, "its machine")
return words{Headline: "No recent backup of " + what, return words{Headline: "No recent backup of " + what,
Needs: fmt.Sprintf("check that backups run on %s.", m),
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+ Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
"changed since would be lost.", conditions.Capital(what), machineOr(o, "its machine")), "changed since would be lost.", conditions.Capital(what), m),
Resolved: conditions.Capital(what) + " is backed up again"} Resolved: conditions.Capital(what) + " is backed up again"}
}), }),
kindArrayDegraded: worded(func(o conditions.Observation) words { kindArrayDegraded: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "Storage on " + m + " is degraded", return words{Headline: "Storage on " + m + " is degraded",
Explanation: fmt.Sprintf("The redundant storage on %s lost a disk or is rebuilding. The data on it is "+ Needs: fmt.Sprintf("check the disks of %s.", m),
"less protected until it is whole again.", m), Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
Resolved: "Storage on " + m + " is whole again"} Resolved: "Storage on " + m + " is whole again"}
}), }),
kindEmptyReplacement: worded(func(o conditions.Observation) words { kindEmptyReplacement: worded(func(o conditions.Observation) words {
module := idPart(o, 1) module := idPart(o, 1)
@@ -282,8 +295,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
module = "a module" module = "a module"
} }
return words{Headline: conditions.Capital(module) + " runs on empty data", return words{Headline: conditions.Capital(module) + " runs on empty data",
Needs: "decide where its data lives: move the data, or move it back.",
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+ Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
"elsewhere. It needs you: move the data, or move it back.", conditions.Capital(module), machineOr(o, "its machine")), "elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
Resolved: conditions.Capital(module) + " has its data again"} Resolved: conditions.Capital(module) + " has its data again"}
}), }),
kindDataHeldTwice: worded(func(o conditions.Observation) words { kindDataHeldTwice: worded(func(o conditions.Observation) words {
@@ -292,40 +306,38 @@ var plainWordings = map[string]func(conditions.Observation) words{
consumer = "a module" consumer = "a module"
} }
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines", return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one. It needs you to "+ Needs: "decide which copy to keep.",
"decide which to keep.", conditions.Capital(consumer)), Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
Resolved: conditions.Capital(consumer) + "'s data is in one place again"} Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
}), }),
// Deliveries and walks. // Deliveries and walks.
"stalled": worded(func(o conditions.Observation) words { "stalled": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeDelivery { if o.Scope == conditions.ScopeDelivery {
return words{Headline: "A delivery is held too long", return words{Headline: "A delivery is held too long",
Explanation: "A delivery has been held past its time limit. " + conditions.ResolverWords(o), Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
Resolved: "Resolved: the delivery moves again"} Resolved: "Resolved: the delivery moves again"}
} }
return words{Headline: "A delivery is stuck halfway", return words{Headline: "A delivery is stuck halfway",
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost; if it " + Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
"stays, an agent can look at why.", Resolved: "Resolved: the delivery moves again"}
Resolved: "Resolved: the delivery moves again"}
}), }),
kindWalkWaiting: worded(func(o conditions.Observation) words { kindWalkWaiting: worded(func(o conditions.Observation) words {
return words{Headline: "A delivery is waiting to start", return words{Headline: "A delivery is waiting to start",
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " + Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
"out) has not said it may start. Nothing to do yet; an agent can start it by hand.", "out) has not let it start yet.",
Resolved: "Resolved: the delivery is no longer waiting"} Resolved: "Resolved: the delivery is no longer waiting"}
}), }),
"merge-not-acted": worded(func(o conditions.Observation) words { "merge-not-acted": worded(func(o conditions.Observation) words {
return words{Headline: "A merge was never picked up", return words{Headline: "A merge was never picked up",
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " + Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
"sent. An agent can hand it over again.", "sent. The mesh catches up on missed merges by itself.",
Resolved: "Resolved: the merge was picked up"} Resolved: "Resolved: the merge was picked up"}
}), }),
"ask-lost": worded(func(o conditions.Observation) words { "ask-lost": worded(func(o conditions.Observation) words {
return words{Headline: "A build has no result", return words{Headline: "A build has no result",
Explanation: "A build was asked for and never answered. What it was for is not delivered until it is " + Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
"built again.", Resolved: "Resolved: the build has its result"}
Resolved: "Resolved: the build has its result"}
}), }),
kindRolledBack: worded(func(o conditions.Observation) words { kindRolledBack: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
@@ -336,68 +348,73 @@ var plainWordings = map[string]func(conditions.Observation) words{
if o.Scope == conditions.ScopeCore { if o.Scope == conditions.ScopeCore {
return words{Headline: "A core update was put back on " + m, return words{Headline: "A core update was put back on " + m,
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+ Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
"before was put back. It needs a look now.", module, m), "before was put back and runs.", module, m),
Resolved: "Resolved: the core on " + m + " is healthy"} Resolved: "Resolved: the core on " + m + " is healthy"}
} }
return words{Headline: conditions.Capital(module) + " update put back on " + m, return words{Headline: conditions.Capital(module) + " update put back on " + m,
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+ Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
"before. The change is not delivered; it needs a fix in its code.", module, m), "before, which runs. The change is not delivered until it is fixed.", module, m),
Resolved: "Resolved: " + module + " on " + m + " is settled"} Resolved: "Resolved: " + module + " on " + m + " is settled"}
}), }),
kindRollbackFailed: worded(func(o conditions.Observation) words { kindRollbackFailed: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "Putting back an update failed on " + m, return words{Headline: "Putting back an update failed on " + m,
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before. What runs "+ Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
"there may be broken; it needs you now.", m), Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
Resolved: "Resolved: " + m + " runs a good build again"} Resolved: "Resolved: " + m + " runs a good build again"}
}), }),
"release-held": worded(func(o conditions.Observation) words { "release-held": worded(func(o conditions.Observation) words {
return words{Headline: "Updates wait for your release", return words{Headline: "Updates wait for your release",
Explanation: "Some module updates wait for a person to release them. They are not delivered until you do.", Needs: "release them, or leave them held.",
Explanation: "Some module updates wait for a person to release them, and are not delivered until then.",
Resolved: "Resolved: the held updates are released"} Resolved: "Resolved: the held updates are released"}
}), }),
"facts-stale": worded(func(o conditions.Observation) words { "facts-stale": worded(func(o conditions.Observation) words {
return words{Headline: "Merge checks use outdated facts", return words{Headline: "Merge checks use outdated facts",
Explanation: "The facts a merge check judges a change against are out of date, so a check may pass or " + Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
"fail for the wrong reason.", "by themselves.",
Resolved: "Merge checks use current facts again"} Resolved: "Merge checks use current facts again"}
}), }),
// The controller and the core. // The controller and the core.
"controller-deaf": worded(func(o conditions.Observation) words { "controller-deaf": worded(func(o conditions.Observation) words {
return words{Headline: "The controller stopped listening", return words{Headline: "The controller stopped listening",
Needs: "restart the controller if this stays.",
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " + Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
"wait. Changes and repairs do not happen until it recovers.", "wait. Changes and repairs do not happen until it recovers.",
Resolved: "The controller listens again"} Resolved: "The controller listens again"}
}), }),
"self-check-silent": worded(func(o conditions.Observation) words { "self-check-silent": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh's self-check stopped", return words{Headline: "The mesh's self-check stopped",
Needs: "restart the controller if this stays.",
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " + Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
"Problems may go unnoticed until it runs again.", "Problems may go unnoticed until it runs again.",
Resolved: "The self-check runs again"} Resolved: "The self-check runs again"}
}), }),
"watchdogs-silent": worded(func(o conditions.Observation) words { "watchdogs-silent": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh's watchdogs stopped", return words{Headline: "The mesh's watchdogs stopped",
Explanation: "The watchdogs, which notice when something expected does not happen, have not run. Missed " + Needs: "restart the controller if this stays.",
"signals are not being noticed.", Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
"missed signals are not noticed.",
Resolved: "The watchdogs run again"} Resolved: "The watchdogs run again"}
}), }),
"lease-lost": worded(func(o conditions.Observation) words { "lease-lost": worded(func(o conditions.Observation) words {
return words{Headline: "The controller lost its lease", return words{Headline: "The controller lost its lease",
Needs: "make sure only one controller runs; the details say which machines claim it.",
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " + Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
"not renew it, so two could act at once, or none. It needs a look now.", "not renew it, so two could act at once, or none.",
Resolved: "The controller holds its lease again"} Resolved: "The controller holds its lease again"}
}), }),
"lease-split": worded(func(o conditions.Observation) words { "lease-split": worded(func(o conditions.Observation) words {
return words{Headline: "Two controllers may be acting", return words{Headline: "Two controllers may be acting",
Explanation: "The lease that lets one controller act at a time does not agree with its record, so two " + Needs: "make sure only one controller runs; the details say which machines claim it.",
"could act at once. It needs a look now.", Explanation: "The lease that lets one controller act at a time does not agree with its record.",
Resolved: "One controller acts again"} Resolved: "One controller acts again"}
}), }),
"stale-writer": worded(func(o conditions.Observation) words { "stale-writer": worded(func(o conditions.Observation) words {
return words{Headline: "Outdated instructions are being sent", return words{Headline: "Outdated instructions are being sent",
Explanation: "Something kept sending instructions older than what the machines already hold, and they " + Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
"were refused. It is harmless if it stops.", "were refused, which is harmless.",
Resolved: "Resolved: no more outdated instructions"} Resolved: "Resolved: no more outdated instructions"}
}), }),
"status-slow": worded(func(o conditions.Observation) words { "status-slow": worded(func(o conditions.Observation) words {
@@ -408,32 +425,32 @@ var plainWordings = map[string]func(conditions.Observation) words{
kindCoreUnhealthy: worded(func(o conditions.Observation) words { kindCoreUnhealthy: worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeBus { if o.Scope == conditions.ScopeBus {
return words{Headline: "The bus is not healthy", return words{Headline: "The bus is not healthy",
Needs: "check the machine the bus runs on; the details say what fails.",
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
"and tool calls may fail until it recovers.", "and tool calls may fail until it recovers.",
Resolved: "The bus is healthy again"} Resolved: "The bus is healthy again"}
} }
m := machineOr(o, "its machine") m := machineOr(o, "its machine")
return words{Headline: "The mesh's own software fails on " + m, return words{Headline: "The mesh's own software fails on " + m,
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy. What it does there may "+ Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
"not happen until it recovers.", m), Resolved: "The mesh's own software on " + m + " is healthy again"}
Resolved: "The mesh's own software on " + m + " is healthy again"}
}), }),
"call-hung": worded(func(o conditions.Observation) words { "call-hung": worded(func(o conditions.Observation) words {
return words{Headline: "A tool call is hanging", return words{Headline: "A tool call is hanging",
Explanation: "A call to one of the mesh's tools has run far past its time limit. Whatever waits on it is " + Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
"stuck.", Resolved: "Resolved: the tool call ended"}
Resolved: "Resolved: the tool call ended"}
}), }),
"probe-failed": worded(func(o conditions.Observation) words { "probe-failed": worded(func(o conditions.Observation) words {
return words{Headline: "A check of the mesh could not run", return words{Headline: "A check of the mesh could not run",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown, which is not " + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
"the same as fine.", "again every few minutes.",
Resolved: "The check runs again"} Resolved: "The check runs again"}
}), }),
kindHealersBraked: worded(func(o conditions.Observation) words { kindHealersBraked: worded(func(o conditions.Observation) words {
return words{Headline: "All automatic repairs stopped", return words{Headline: "All automatic repairs stopped",
Needs: "read the details and decide whether the repairs may run again.",
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " + Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
"looks like a loop, so all of them stopped. Nothing repairs itself until this is looked at.", "looks like a loop, so all of them stopped.",
Resolved: "Automatic repairs run again"} Resolved: "Automatic repairs run again"}
}), }),
"healer-wanted": worded(func(o conditions.Observation) words { "healer-wanted": worded(func(o conditions.Observation) words {
@@ -442,8 +459,8 @@ var plainWordings = map[string]func(conditions.Observation) words{
cause = "something" cause = "something"
} }
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand", return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
Explanation: "A person keeps repairing this by hand. A healer (an automatic repair) is wanted for it; " + Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
"nothing is broken now.", "broken now.",
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")} Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
}), }),
@@ -451,8 +468,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
"resolver-wrong": worded(func(o conditions.Observation) words { "resolver-wrong": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "Name lookups fail on " + m, return words{Headline: "Name lookups fail on " + m,
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly. "+ Needs: fmt.Sprintf("check that %s is on and online.", m),
"Machines that rely on it may not find each other.", m), Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
"machines that rely on it may not find each other.", m),
Resolved: "Name lookups work on " + m + " again"} Resolved: "Name lookups work on " + m + " again"}
}), }),
"holder-silent": worded(func(o conditions.Observation) words { "holder-silent": worded(func(o conditions.Observation) words {
@@ -468,50 +486,47 @@ var plainWordings = map[string]func(conditions.Observation) words{
}), }),
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words { kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
return words{Headline: "The bus could not be fully set up", return words{Headline: "The bus could not be fully set up",
Explanation: "When instructions were sent, the bus's queues could not all be checked, so a module may " + Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
"not get its messages.", "again with the next send.",
Resolved: "The bus is fully set up again"} Resolved: "The bus is fully set up again"}
}), }),
kindBusMaintenance: worded(func(o conditions.Observation) words { kindBusMaintenance: worded(func(o conditions.Observation) words {
return words{Headline: "The bus is being upgraded", return words{Headline: "The bus is being upgraded",
Explanation: "The mesh's message system is being upgraded; some things pause until it is done. Nothing " + Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
"to do.", Resolved: "The bus upgrade is done"}
Resolved: "The bus upgrade is done"}
}), }),
kindBusUpgradeFailed: worded(func(o conditions.Observation) words { kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
return words{Headline: "The bus upgrade failed", return words{Headline: "The bus upgrade failed",
Explanation: "The upgrade of the mesh's message system did not end healthy in time. It needs you: it can " + Needs: "decide whether to put the bus back to the version before; the details say how.",
"be put back to the version before.", Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
Resolved: "Resolved: the bus is healthy after its upgrade"} Resolved: "Resolved: the bus is healthy after its upgrade"}
}), }),
kindConsumerBehind: worded(func(o conditions.Observation) words { kindConsumerBehind: worded(func(o conditions.Observation) words {
return words{Headline: "Messages pile up for a listener", return words{Headline: "Messages pile up for a listener",
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " + Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
"healer may restart it.", "healer restarts it if it stays.",
Resolved: "The listener caught up"} Resolved: "The listener caught up"}
}), }),
"consumer-wrong": worded(func(o conditions.Observation) words { "consumer-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is set up wrong", return words{Headline: "A listener on the bus is set up wrong",
Explanation: "One of the mesh's listeners on the bus is missing or not as the controller defines it, so " + Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
"some messages may reach nobody.", "it up again.",
Resolved: "The listener is set up right again"} Resolved: "The listener is set up right again"}
}), }),
"consumer-lost": worded(func(o conditions.Observation) words { "consumer-lost": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is gone", return words{Headline: "A listener on the bus is gone",
Explanation: "One of the mesh's listeners is no longer on the bus, so what it should receive reaches " + Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
"nobody.", Resolved: "The listener is back"}
Resolved: "The listener is back"}
}), }),
"slow-consumer": worded(func(o conditions.Observation) words { "slow-consumer": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is too slow", return words{Headline: "A listener on the bus is too slow",
Explanation: "The bus reports a listener too slow to keep up, so messages to it are dropped or late.", Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
Resolved: "The listener keeps up again"} Resolved: "The listener keeps up again"}
}), }),
"max-deliveries": worded(func(o conditions.Observation) words { "max-deliveries": worded(func(o conditions.Observation) words {
return words{Headline: "A message could not be handled", return words{Headline: "A message could not be handled",
Explanation: "The bus gave up on a message after trying to hand it over too many times. What it carried " + Explanation: "The bus gave up on a message after trying to hand it over too many times.",
"did not happen.", Resolved: "Resolved: messages are handled again"}
Resolved: "Resolved: messages are handled again"}
}), }),
"refused": worded(func(o conditions.Observation) words { "refused": worded(func(o conditions.Observation) words {
return words{Headline: "The bus refuses some messages", return words{Headline: "The bus refuses some messages",
@@ -520,21 +535,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
}), }),
"stream-wrong": worded(func(o conditions.Observation) words { "stream-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "Part of the bus's storage is wrong", return words{Headline: "Part of the bus's storage is wrong",
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps " + Needs: "check the machine the bus runs on; the details say what is missing.",
"there may be lost.", Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
Resolved: "The bus's storage is right again"} Resolved: "The bus's storage is right again"}
}), }),
"archives-unheld": worded(func(o conditions.Observation) words { "archives-unheld": worded(func(o conditions.Observation) words {
return words{Headline: "Kept builds are about to be cleaned up", return words{Headline: "Kept builds are about to be cleaned up",
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them. " + Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
"Nothing to do unless one is still needed.", Resolved: "Resolved: every kept build is claimed"}
Resolved: "Resolved: every kept build is claimed"}
}), }),
"archives-missing": worded(func(o conditions.Observation) words { "archives-missing": worded(func(o conditions.Observation) words {
return words{Headline: "Kept builds are missing", return words{Headline: "Kept builds are missing",
Explanation: "Some builds the mesh relies on are no longer in the store. A machine that needs one cannot " + Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
"get it until it is built again.", Resolved: "Resolved: the kept builds are back"}
Resolved: "Resolved: the kept builds are back"}
}), }),
} }
@@ -602,18 +615,68 @@ func deliveryName(modules []string, repository string) string {
return repoName(repository) + " delivery" return repoName(repository) + " delivery"
} }
// waitingAct is what the operator does about a walk waiting for its delivery's word. // walkWaitingWords explains a walk waiting for its delivery's word.
func waitingAct(severity conditions.Severity) string { func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
what := deliveryWhat(w.modules, w.repository)
if severity == conditions.Urgent { if severity == conditions.Urgent {
return "Mesh-delivery may be broken. It needs you: have an agent look at mesh-delivery, or start the " + return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
"delivery by hand." "delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
} }
return fmt.Sprintf("Nothing to do yet; it becomes urgent after %s. To start it now, have an agent start it by hand.", return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
humanDuration(waitUrgentAfter)) "decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
what, humanDuration(in), humanDuration(waitUrgentAfter))
}
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
func waitingNeeds(severity conditions.Severity) string {
if severity == conditions.Urgent {
return "start it, or stop it."
}
return ""
}
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
// the module that should have said go is the one not answering.
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
if severity != conditions.Urgent {
return nil
}
return []conditions.Action{
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
}
}
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
var actions []conditions.Action
for _, r := range rs {
if strings.Contains(r.Reason, "relogin needed") {
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
}
if r.Kind == link.KindUnit && len(actions) < 2 {
scope := "system"
if strings.Contains(r.Reason, "account's own") {
scope = "user"
}
label := "Restart"
if len(actions) > 0 {
label = "Restart " + unitPlainWords(r.Target)
}
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
}
}
if len(actions) > 0 {
return "restart it; if it fails again, the details say why.", actions
}
return "", nil
} }
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it. // stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
func stalledWords(l stalledLine, o conditions.Observation) (string, string, string) { func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
actions []conditions.Action) {
repository, _, _ := strings.Cut(l.ID, "@") repository, _, _ := strings.Cut(l.ID, "@")
name := repoName(repository) name := repoName(repository)
held := l.State held := l.State
@@ -624,13 +687,24 @@ func stalledWords(l stalledLine, o conditions.Observation) (string, string, stri
if d, err := time.ParseDuration(l.For); err == nil { if d, err := time.ParseDuration(l.For); err == nil {
long = "for " + humanDuration(d) long = "for " + humanDuration(d)
} }
act := conditions.ResolverWords(o)
if o.Resolver == conditions.ResolverOperator { if o.Resolver == conditions.ResolverOperator {
act = "It waits for you: release it, or stop it." switch held {
case "held":
needs = "release it, or stop it."
actions = []conditions.Action{
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
}
case "ready", "checked":
needs = "merge its pull request, or close it."
default:
needs = "stop it, or read the details to see what it waits for."
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
}
} }
return fmt.Sprintf("Delivery of %s %s %s", name, held, long), return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
fmt.Sprintf("A delivery of %s has been %s %s, past its limit. %s", name, held, long, act), fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
fmt.Sprintf("Delivery of %s is no longer %s", name, held) fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
} }
// causeWords is a hand-act's cause as a person says it. // causeWords is a hand-act's cause as a person says it.
+74 -30
View File
@@ -14,29 +14,40 @@ import (
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a // each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them. // resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
// plainExample checks one finding's words: plain, within their bounds, and saying what the operator needs. // plainExample checks one finding's words as the keeper will keep them: plain, within their bounds,
func plainExample(t *testing.T, o conditions.Observation, headline string, says ...string) { // opened by the verdict, and saying what the operator needs; and its actions by label.
func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) {
t.Helper() t.Helper()
machines := append([]string{o.Machine}, o.Also...) machines := append([]string{o.Machine}, o.Also...)
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved} w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
Actions: o.Actions}
if why, ok := conditions.PlainWords(w, machines...); !ok { if why, ok := conditions.PlainWords(w, machines...); !ok {
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w) t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
} }
if o.Headline != headline { said := conditions.Verdict(o.Needs, o.Explanation)
t.Errorf("%s: headline %q, want %q", o.Key(), o.Headline, headline) if o.Headline != headline || said != explanation {
t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation)
} }
for _, s := range says { var labels []string
if !strings.Contains(o.Explanation+" | "+o.Resolved, s) { for _, a := range o.Actions {
t.Errorf("%s: the words do not say %q:\n %s | %s", o.Key(), s, o.Explanation, o.Resolved) labels = append(labels, a.Label)
}
} }
t.Logf("\nBEFORE: %s\nAFTER: %s\n %s\nCLEARS: %s", o.Summary, o.Headline, o.Explanation, o.Resolved) if strings.Join(labels, ",") != strings.Join(actions, ",") {
t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions)
}
quiet := ""
if o.Needs == "" && o.Severity == conditions.Warning {
quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)"
}
t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet,
said, labels, o.Resolved)
} }
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for // **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why; // mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
// `plans go plan-1791454185265004861 --why …` starts it by hand". // `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no
func TestADeliveryWaitingIsSaidByItsModulesAndWhatToDo(t *testing.T) { // popup; past them, the operator starts or stops it from the notification.
func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) {
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC) now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
f := calm(now) f := calm(now)
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee", f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
@@ -45,36 +56,59 @@ func TestADeliveryWaitingIsSaidByItsModulesAndWhatToDo(t *testing.T) {
if len(got) != 1 { if len(got) != 1 {
t.Fatalf("%+v", got) t.Fatalf("%+v", got)
} }
plainExample(t, got[0], "openrazer delivery waiting to start", "56 minutes", "mesh-delivery (the module that decides", plainExample(t, got[0], "openrazer delivery waiting to start",
"Nothing to do yet", "openrazer delivery no longer waiting") "Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+
"for you if it still waits after 4 hours.")
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") { if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary) t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
} }
// Past four hours it is urgent, and says it needs the operator. // Past four hours it is urgent, and offers the controller's own answers.
f.waits[0].since = now.Add(-5 * time.Hour) f.waits[0].since = now.Add(-5 * time.Hour)
got = watchWaits(f) got = watchWaits(f)
plainExample(t, got[0], "openrazer delivery waiting to start", "It needs you") plainExample(t, got[0], "openrazer delivery waiting to start",
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
"be stuck.", "Start", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
t.Errorf("start: %+v", a)
}
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
t.Errorf("stop: %+v", a)
}
// Many modules are counted, not listed in the headline. // Many modules are counted, not listed in the headline.
f.waits[0].modules = []string{"a", "b", "c", "d"} f.waits[0].modules = []string{"a", "b", "c", "d"}
got = watchWaits(f) got = watchWaits(f)
plainExample(t, got[0], "mesh-catalog delivery waiting to start", "4 modules of mesh-catalog") if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") {
t.Errorf("%+v", got[0])
}
} }
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the // **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
// account's own service manager (exit-code)". // account's own service manager (exit-code)". The operator restarts it from the notification.
func TestAModuleUnhealthyNamesItsServiceInWords(t *testing.T) { func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit, o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
Resource: "openrazer-daemon", Target: "openrazer-daemon.service", Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}}) Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
plainExample(t, o, "openrazer not working on g14", "is not healthy: its service openrazer-daemon stopped with an error", plainExample(t, o, "openrazer not working on g14",
"openrazer works again on g14") "Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
t.Errorf("restart: %+v", a)
}
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
}
} }
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places — // **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). Each is the // mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …"
// machine's own: mend or remove it there, or have a module place it".
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) { func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{ o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"}, {Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
@@ -83,12 +117,14 @@ func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
t.Fatal("not raised") t.Fatal("not raised")
} }
plainExample(t, o, "3 failed services on shanks", plainExample(t, o, "3 failed services on shanks",
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed", "mend or remove them on shanks") "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+
"mesh does not repair them.", "Silence for a week")
} }
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, // **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through
// through the mesh-controller seat: a healer is wanted for it". // the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup.
func TestAHealerWantedSaysNothingIsBroken(t *testing.T) { func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
f := calm(now) f := calm(now)
for i := 0; i < 35; i++ { for i := 0; i < 35; i++ {
@@ -100,15 +136,22 @@ func TestAHealerWantedSaysNothingIsBroken(t *testing.T) {
if len(got) != 1 { if len(got) != 1 {
t.Fatalf("%+v", got) t.Fatalf("%+v", got)
} }
plainExample(t, got[0], "Push keeps being fixed by hand", "35 times in 14 days", "nothing is broken now") plainExample(t, got[0], "Push keeps being fixed by hand",
"Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+
"wanted for it. Nothing is broken now.")
} }
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been // **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …". // held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
func TestADeliveryHeldSaysItWaitsForTheOperator(t *testing.T) { func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}}) Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
plainExample(t, got[0], "Delivery of hq held for 36 hours", "It waits for you: release it, or stop it.") plainExample(t, got[0], "Delivery of hq held for 36 hours",
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
"Release", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
t.Errorf("release: %+v", a)
}
} }
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every // **Every kind the controller raises has plain words**, and its words are plain for a subject of every
@@ -134,6 +177,7 @@ func TestEveryWordingIsPlain(t *testing.T) {
for _, s := range subjects { for _, s := range subjects {
s.Kind = kind s.Kind = kind
w := fn(s) w := fn(s)
w.Explanation = conditions.Verdict(w.Needs, w.Explanation)
if why, ok := conditions.PlainWords(w, s.Machine); !ok { if why, ok := conditions.PlainWords(w, s.Machine); !ok {
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w) t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
} }
+9 -9
View File
@@ -324,7 +324,7 @@ func watchPlans(f *signalFacts) []conditions.Observation {
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled", out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway", Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway",
Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+ Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+
"usual. Nothing is lost; if it stays, an agent can look at why.", humanDuration(f.now.Sub(p.entered))), "usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))),
Resolved: "Delivery of " + repoName(p.repository) + " moves again", Resolved: "Delivery of " + repoName(p.repository) + " moves again",
Severity: conditions.Warning, Severity: conditions.Warning,
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s", Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
@@ -357,12 +357,12 @@ func watchWaits(f *signalFacts) []conditions.Observation {
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+ Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository, "delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
short(w.commit), ago(in), w.awaits, short(w.commit), w.id), short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits), Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
Headline: deliveryName(w.modules, w.repository) + " waiting to start", Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the "+ Explanation: walkWaitingWords(w, in, severity),
"module that decides when a delivery goes out) to let it start. %s", deliveryWhat(w.modules, w.repository), Needs: waitingNeeds(severity),
humanDuration(in), waitingAct(severity)), Actions: waitingActions(w, severity),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"}) Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
} }
return out return out
} }
@@ -692,8 +692,8 @@ func watchHandActs(f *signalFacts) []conditions.Observation {
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted), repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
Said: strings.Join(acts, "; "), Said: strings.Join(acts, "; "),
Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand", Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand",
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days. A healer (an automatic repair) "+ Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is "+
"is wanted for it; nothing is broken now.", causeWords(cause), repeated[cause], "wanted for it. Nothing is broken now.", causeWords(cause), repeated[cause],
int(handActsWithin.Hours()/24)), int(handActsWithin.Hours()/24)),
Resolved: "Resolved: no more hand repairs of " + causeWords(cause)}) Resolved: "Resolved: no more hand repairs of " + causeWords(cause)})
} }
+6
View File
@@ -134,6 +134,10 @@ type Condition struct {
Headline string `json:"headline"` Headline string `json:"headline"`
Explanation string `json:"explanation"` Explanation string `json:"explanation"`
Resolved string `json:"resolved"` Resolved string `json:"resolved"`
// Needs is what the operator does about it, one sentence; empty when nothing (the explanation then
// opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go).
Needs string `json:"needs"`
Actions []Action `json:"actions"`
// Evidence is the newest observations, at most KeptEvidence, newest first. // Evidence is the newest observations, at most KeptEvidence, newest first.
Evidence []Evidence `json:"evidence"` Evidence []Evidence `json:"evidence"`
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. // Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
@@ -187,6 +191,8 @@ type Observation struct {
Headline string Headline string
Explanation string Explanation string
Resolved string Resolved string
Needs string
Actions []Action
// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes // Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
// here, never in Summary**: an address, a socket's error, a path or a name with its domain is // here, never in Summary**: an address, a socket's error, a path or a name with its domain is
// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the // kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
+91 -23
View File
@@ -11,7 +11,12 @@ package conditions
// g14"), the title of every message about it; // g14"), the title of every message about it;
// - an **explanation**: one or two plain sentences — what happened, what it means for the operator, // - an **explanation**: one or two plain sentences — what happened, what it means for the operator,
// and whether they need to do anything; // and whether they need to do anything;
// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14"). // - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14");
// - a **verdict**, which every explanation opens with: "Nothing for you to do." or "Needs you:" and one
// concrete thing the operator can do themselves (needs). Never "have an agent …": the operator is not
// asked to open a session to understand or answer a notification;
// - **actions**: what the operator may answer from the notification itself (Release, Stop, Restart,
// Silence), each the seat verb the operator channel calls when it is chosen, naming the operator.
// //
// They are made here, where the condition is made, so every channel gets them — the desktop today, // They are made here, where the condition is made, so every channel gets them — the desktop today,
// others later — and none has to guess what a key means. A producer may say them itself; otherwise the // others later — and none has to guess what a key means. A producer may say them itself; otherwise the
@@ -29,17 +34,45 @@ import (
"github.com/novox/mesh-controller/internal/outward" "github.com/novox/mesh-controller/internal/outward"
) )
// Words are what the operator reads of a condition. // Words are what the operator reads of a condition. Explanation is said after the verdict, which
// plainly writes from Needs: empty is "Nothing for you to do.", else "Needs you: " and Needs.
type Words struct { type Words struct {
Headline string Headline string
Explanation string Explanation string
Resolved string Resolved string
Needs string
Actions []Action
}
// Action is one answer the operator may give from a notification: a label, and the seat verb the
// operator channel calls with these arguments (and a why naming the operator and the label) when it is
// chosen. Machine is set for a seat every machine holds. An argument "why" given empty is the operator
// channel's to fill: it names the operator, the channel and the label chosen.
type Action struct {
Label string `json:"label"`
Verb string `json:"verb"`
Machine string `json:"machine,omitempty"`
Arguments map[string]string `json:"arguments,omitempty"`
}
// The two verdicts an explanation opens with.
const (
NothingToDo = "Nothing for you to do."
NeedsYou = "Needs you:"
)
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
// answer to a condition the operator decided to live with.
func SilenceAction(key string) Action {
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
} }
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences. // Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
const ( const (
HeadlineMax = 60 HeadlineMax = 60
ExplanationMax = 360 ExplanationMax = 420
NeedsMax = 120
) )
var ( var (
@@ -75,6 +108,7 @@ var (
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`) goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`) clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
flag = regexp.MustCompile(`(^|\s)--?[a-z]`) flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)") markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
) )
@@ -124,6 +158,27 @@ func PlainWords(w Words, machines ...string) (string, bool) {
if len(w.Explanation) > ExplanationMax { if len(w.Explanation) > ExplanationMax {
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
} }
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
return "explanation: sends the operator elsewhere (" + m + ")", false
}
}
if w.Needs != "" {
if why, ok := Plain(w.Needs, machines...); !ok {
return "needs: " + why, false
}
if m := agentWord.FindString(w.Needs); m != "" {
return "needs: not something the operator does themselves (" + m + ")", false
}
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
return fmt.Sprintf("needs: one sentence of at most %d characters, ending in a full stop", NeedsMax), false
}
}
for _, a := range w.Actions {
if a.Label == "" || len(a.Label) > 24 || a.Verb == "" {
return fmt.Sprintf("action %q: a short label and a verb", a.Label), false
}
}
if why, ok := Plain(w.Resolved, machines...); !ok { if why, ok := Plain(w.Resolved, machines...); !ok {
return "resolved: " + why, false return "resolved: " + why, false
} }
@@ -136,7 +191,8 @@ func PlainWords(w Words, machines ...string) (string, bool) {
// plainly gives an observation its plain words: its own, its kind's, or its scope's. // plainly gives an observation its plain words: its own, its kind's, or its scope's.
func plainly(o Observation) Observation { func plainly(o Observation) Observation {
machines := append([]string{o.Machine}, o.Also...) machines := append([]string{o.Machine}, o.Also...)
given := Words{o.Headline, o.Explanation, o.Resolved} given := Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
Actions: o.Actions}
var w Words var w Words
from := "" from := ""
switch { switch {
@@ -154,6 +210,12 @@ func plainly(o Observation) Observation {
if given.Resolved != "" { if given.Resolved != "" {
w.Resolved = given.Resolved w.Resolved = given.Resolved
} }
if given.Needs != "" {
w.Needs = given.Needs
}
if len(given.Actions) > 0 {
w.Actions = given.Actions
}
} }
} }
if w.Headline != "" && w.Resolved == "" { if w.Headline != "" && w.Resolved == "" {
@@ -170,17 +232,27 @@ func plainly(o Observation) Observation {
} }
w = scopeWords(o) w = scopeWords(o)
} }
o.Headline, o.Explanation, o.Resolved = w.Headline, w.Explanation, w.Resolved o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
o.Explanation = Verdict(w.Needs, w.Explanation)
return o return o
} }
// Escalated is the sentence a condition's explanation ends with once a healer gave up on it. // Verdict is an explanation opened by its verdict.
const Escalated = "The mesh tried to repair it and could not: it needs you now." func Verdict(needs, explanation string) string {
if needs == "" {
return strings.TrimSpace(NothingToDo + " " + explanation)
}
return strings.TrimSpace(NeedsYou + " " + needs + " " + explanation)
}
// escalatedWords says in the explanation that a healer gave up, whatever the words said before. // Escalated is what a condition needs once a healer gave up on it, where its words needed nothing.
const Escalated = "the mesh tried to repair this and could not; read the details to decide what to do."
// escalatedWords turns "Nothing for you to do." into "Needs you:" once a healer gave up.
func escalatedWords(c *Condition) { func escalatedWords(c *Condition) {
if c.Escalated() && !strings.HasSuffix(c.Explanation, Escalated) { if c.Escalated() && c.Needs == "" {
c.Explanation = strings.TrimSpace(c.Explanation + " " + Escalated) c.Needs = Escalated
c.Explanation = Verdict(c.Needs, strings.TrimSpace(strings.TrimPrefix(c.Explanation, NothingToDo)))
} }
} }
@@ -191,13 +263,14 @@ func scopeWords(o Observation) Words {
what := strings.ReplaceAll(o.Kind, "-", " ") what := strings.ReplaceAll(o.Kind, "-", " ")
w := Words{ w := Words{
Headline: Capital(thing) + " needs a look", Headline: Capital(thing) + " needs a look",
Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s. %s", what, thing, ResolverWords(o)), Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s.", what, thing),
Resolved: "Resolved: " + thing + " is fine again", Resolved: "Resolved: " + thing + " is fine again",
Needs: ResolverNeeds(o),
} }
if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok { if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok {
w = Words{Headline: "Something in the mesh needs a look", w = Words{Headline: "Something in the mesh needs a look",
Explanation: "The mesh noticed a problem it has no plain words for yet. " + ResolverWords(o), Explanation: "The mesh noticed a problem it has no plain words for yet.",
Resolved: "Resolved: the mesh is fine again"} Resolved: "Resolved: the mesh is fine again", Needs: ResolverNeeds(o)}
} }
return w return w
} }
@@ -244,17 +317,12 @@ func ThingWords(o Observation) string {
return "the mesh" return "the mesh"
} }
// ResolverWords says whether the operator needs to act, from who resolves it. // ResolverNeeds is what the operator needs to do, from who resolves it: nothing, unless only a person can.
func ResolverWords(o Observation) string { func ResolverNeeds(o Observation) string {
switch { if o.Resolver == ResolverOperator {
case o.Resolver == ResolverOperator: return "the mesh does not repair this by itself; read the details to decide what to do."
return "It needs you: the mesh does not repair this by itself."
case strings.HasPrefix(o.Resolver, "healer:"):
return "A healer is working on it; nothing to do unless it stays."
case o.Severity == Urgent:
return "It clears by itself once it is fixed, but needs a look now."
} }
return "Nothing to do yet: it clears by itself once it is fixed." return ""
} }
// Capital is s with its first letter upper case. // Capital is s with its first letter upper case.
+32 -6
View File
@@ -79,7 +79,7 @@ func TestAConditionCarriesItsPlainWords(t *testing.T) {
} }
said := settled(t, told, 1) said := settled(t, told, 1)
body, _ := json.Marshal(said[0]) body, _ := json.Marshal(said[0])
for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"The mesh has not heard from it."`, for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"Nothing for you to do. The mesh has not heard from it."`,
`"resolved":"ace answers again"`} { `"resolved":"ace answers again"`} {
if !strings.Contains(string(body), field) { if !strings.Contains(string(body), field) {
t.Errorf("the event does not carry %s: %s", field, body) t.Errorf("the event does not carry %s: %s", field, body)
@@ -100,7 +100,7 @@ func TestAConditionCarriesItsPlainWords(t *testing.T) {
// Words that are not plain, and a kind with none, are said from the scope — and reported. // Words that are not plain, and a kind with none, are said from the scope — and reported.
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded", c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded",
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"}) Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"})
if c.Headline != "Openrazer on g14 needs a look" || !strings.Contains(c.Explanation, "Nothing to do yet") { if c.Headline != "Openrazer on g14 needs a look" || !strings.HasPrefix(c.Explanation, NothingToDo) {
t.Errorf("the scope's words: %+v", c) t.Errorf("the scope's words: %+v", c)
} }
c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent", c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent",
@@ -113,13 +113,39 @@ func TestAConditionCarriesItsPlainWords(t *testing.T) {
} }
} }
// **An explanation opens with its verdict, and never sends the operator to an agent.**
func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
if got := Verdict("", "It may be asleep."); got != "Nothing for you to do. It may be asleep." {
t.Errorf("%q", got)
}
if got := Verdict("release it, or stop it.", "It is held."); got != "Needs you: release it, or stop it. It is held." {
t.Errorf("%q", got)
}
for _, w := range []Words{
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
} {
if _, ok := PlainWords(w); ok {
t.Errorf("passed: %+v", w)
}
}
if why, ok := PlainWords(Words{Headline: "push keeps being fixed by hand", Explanation: "A person repaired " +
"push by hand 35 times.", Resolved: "Resolved", Needs: "release it, or stop it.",
Actions: []Action{{Label: "Release", Verb: "mesh-delivery.release"}}}); !ok {
t.Errorf("refused: %s", why)
}
}
// **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the // **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the
// budget is spent, the operator is needed. // budget is spent, the operator is needed.
func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) { func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
k, _, _, _ := keeper(t) k, _, _, _ := keeper(t)
ctx := t.Context() ctx := t.Context()
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1", o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "Nothing to do yet."} Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
if _, err := k.Observe(ctx, o); err != nil { if _, err := k.Observe(ctx, o); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -127,11 +153,11 @@ func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !strings.HasSuffix(c.Explanation, Escalated) { if want := "Needs you: " + Escalated + " It may be asleep."; c.Explanation != want || c.Needs != Escalated {
t.Fatalf("escalated: %q", c.Explanation) t.Fatalf("escalated: %q, want %q", c.Explanation, want)
} }
c, _ = k.Observe(ctx, o) c, _ = k.Observe(ctx, o)
if strings.Count(c.Explanation, Escalated) != 1 { if strings.Count(c.Explanation, Escalated) != 1 || strings.Contains(c.Explanation, NothingToDo) {
t.Errorf("seen again after escalation: %q", c.Explanation) t.Errorf("seen again after escalation: %q", c.Explanation)
} }
} }
+2 -2
View File
@@ -190,7 +190,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if !found { if !found {
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also}, c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation, Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved, Evidence: []Evidence{{At: now, Said: said}}, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}},
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1, Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
Resolver: orSelf(o.Resolver)} Resolver: orSelf(o.Resolver)}
change := ChangeRaised change := ChangeRaised
@@ -240,7 +240,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
// The kind as the source says it now: a source that gave the same key a kind of its own since // The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation. // (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
c.Headline, c.Explanation, c.Resolved = o.Headline, o.Explanation, o.Resolved c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
escalatedWords(&c) escalatedWords(&c)
if o.Machine != "" { if o.Machine != "" {
c.Subject.Machine = o.Machine c.Subject.Machine = o.Machine