Compare commits

..
6 Commits
Author SHA1 Message Date
jochen 51d7bbfdc8 Give relogin-needed its plain words: it needs the operator and offers no button, since only their new login can end it (hq ADR 0253, ADR 0254)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/318-a-wait-for-a-person-is-not-a-failure delivered: every member is delivered
2026-10-08 14:25:20 +02:00
jochen e3b5c224e8 Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318) 2026-10-08 14:20:55 +02:00
mesh-admin 6df30b6d98 Merge pull request 'Give every condition a headline, an explanation and a resolved line in plain words (hq ADR 0253, issue 320)' (#141) from feat/plain-notifications into main 2026-10-08 12:20:31 +00:00
jochen 1a4305213d Open every explanation with what the operator needs to do, and offer the answers
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/plain-notifications delivered: every member is delivered
The operator could not tell from a notification whether to act, and was told to
have an agent do it. Each condition now says "Nothing for you to do." or
"Needs you:" with one thing they can do themselves, and carries the actions the
operator channel performs when chosen (hq ADR 0253).
2026-10-08 13:58:53 +02:00
jochen 7df72edd2b Resolve the forge's manifest against both artifacts it now builds
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery-group group feat/plain-notifications ready: every member ready, and composed together they pass
mesh/delivery superseded: a newer head of the same pull request
The catalogue's forge also builds its npm registry since hq ADR 0251, so the three
forge tests resolved it against half its build and failed on main as on every branch.
2026-10-08 13:35:10 +02:00
jochen 1fce541023 Give every condition a headline, an explanation and a resolved line in plain words
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: --- FAIL: TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt (0.00s)
mesh/delivery-group group feat/plain-notifications rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
The operator could not read the desktop notifications: they carried plan ids,
commits, keys and verb syntax. The words the operator reads now travel with the
condition, so every channel says them (hq ADR 0253).
2026-10-08 13:22:27 +02:00
13 changed files with 1622 additions and 10 deletions
@@ -113,6 +113,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
if l.operatorsOnly() {
o.Resolver = conditions.ResolverOperator
}
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
out = append(out, o)
}
return out
+12 -1
View File
@@ -90,8 +90,9 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
if len(u.Failed) == 0 {
return o, false
}
var names, said []string
var names, said, plain []string
for _, f := range u.Failed {
plain = append(plain, unitPlainWords(f.Unit))
where := ""
if f.Scope == "user" {
where = " (the account's own manager)"
@@ -111,6 +112,16 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
"Each is the machine's own: mend or remove it there, or have a module place it",
node, len(u.Failed), strings.Join(names, ", "))
o.Said = strings.Join(said, "; ")
failed := "1 failed service"
if len(u.Failed) > 1 {
failed = fmt.Sprintf("%d failed services", len(u.Failed))
}
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
node, namesWords(plain, 3))
o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
o.Resolved = "No failed services on " + node + " any more"
return o, true
}
+27 -4
View File
@@ -180,6 +180,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
}
}
seen[o.Key()] = true
@@ -255,18 +256,34 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
}
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence.
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
// person in again, and a restart of the module's service would start it in the same session.
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
o := moduleUnhealthyObservation(module, node, rs)
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
w := reloginWords(module, node)
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
return o
}
// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
func reloginWords(module, node string) words {
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
}
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
var words, said []string
var words, said, plain []string
needs, actions := moduleNeeds(node, rs)
for _, r := range rs {
plain = append(plain, resourcePlainWords(r))
if r.Kind == link.KindUnit {
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
// resource that places it — a package, a file — is evidence.
@@ -282,8 +299,14 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
}
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
Said: strings.Join(said, "; ")}
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
Said: strings.Join(said, "; "),
Headline: fmt.Sprintf("%s not working on %s", module, node),
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
namesWords(plain, 3)),
Needs: needs,
Actions: actions,
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
}
// reasonWords is why a resource is unhealthy, as a person reads it.
+19
View File
@@ -193,3 +193,22 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
t.Errorf("the walk does not say what kept its pass: %s", p.Note)
}
}
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
// no button, since nothing but their own new login can do it.
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
plainExample(t, o, "openrazer waits for a new login on g14",
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
"group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+
"update is in place and nothing was undone.")
if len(o.Actions) != 0 || o.Needs == "" {
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
}
// The kind's own wording, for a condition raised without words, says the same.
if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs ||
w.Headline != o.Headline || len(w.Actions) != 0 {
t.Errorf("the kind's wording: %+v", w)
}
}
+776
View File
@@ -0,0 +1,776 @@
package main
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
//
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
// suite is held to having words, and to their being plain (plain_words_test.go).
import (
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
type words = conditions.Words
// worded is a wording that needs only the condition's subject.
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
func init() {
for kind, fn := range plainWordings {
conditions.Wording(kind, fn)
}
}
// machineOr is the machine a condition concerns, or what to say when none is named.
func machineOr(o conditions.Observation, none string) string {
if o.Machine != "" {
return o.Machine
}
return none
}
// idPart is the n-th dotted part of a condition's id, or "".
func idPart(o conditions.Observation, n int) string {
parts := strings.Split(o.ID, ".")
if n < len(parts) {
return parts[n]
}
return ""
}
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
func dataWords(o conditions.Observation) (what, module string) {
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
switch {
case module != "" && item != "":
return item + " of " + module, module
case module != "":
return "data of " + module, module
}
return "data", "a module"
}
var plainWordings = map[string]func(conditions.Observation) words{
// Machines and their node-engine.
"silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
w := words{Headline: m + " is not answering",
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
"normal.", m),
Resolved: m + " answers again"}
if o.Severity == conditions.Urgent {
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
"nothing changes anywhere until it is back.", m)
}
return w
}),
"sent-not-reported": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " has not confirmed a change",
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
"It usually catches up by itself.", m, m),
Resolved: m + " applied the change"}
}),
"tools-silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Tools on " + m + " are not answering",
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
"the mesh until it does. A machine asleep is normal.", m, m),
Resolved: "Tools on " + m + " answer again"}
}),
kindAwaitingPush: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " waits for new instructions",
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
conditions.Capital(m)),
Resolved: m + " got its new instructions"}
}),
"declaration-refused": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeMesh {
return words{Headline: "No machine can get new instructions",
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
"made. Machines keep running what they have.",
Resolved: "Machines can get new instructions again"}
}
m := machineOr(o, "a machine")
return words{Headline: m + " cannot get new instructions",
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
"reaches it. It keeps running what it has.", m),
Resolved: m + " can get new instructions again"}
}),
"own-address-banned": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " has banned the mesh",
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
"mesh is locked out of it.", m),
Resolved: m + " no longer bans the mesh"}
}),
"core-behind": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " runs an old node-engine",
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
Resolved: m + " runs the current node-engine"}
}),
kindMachineUnits: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Failed services on " + m,
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
Explanation: "No module manages them, so the mesh does not repair them.",
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}),
kindMachineNetwork: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " has a network problem",
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
"route. It often passes again once a network change settles.", conditions.Capital(m)),
Resolved: m + "'s network is fine again"}
}),
kindNetworkRewritten: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + "'s network settings were changed",
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
"client. Mesh names may not resolve there until it is undone.", m),
Resolved: m + "'s network settings are the mesh's again"}
}),
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " cannot be reached",
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
"asleep is normal.", m),
Resolved: m + " can be reached again"}
}),
kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m,
Needs: "decide where its data lives; the details name both places.",
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
"kept the old one because switching would leave the data behind.", m),
Resolved: "Resolved: the data source on " + m + " is decided"}
}),
kindBindingMoving: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source moves on " + m,
Needs: "move its data to the new provider before the next push.",
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"}
}),
kindBindingMoved: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source moved on " + m,
Needs: "check that it found its data.",
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
Resolved: "Resolved: the data source move on " + m + " is settled"}
}),
// Modules and providers.
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: conditions.Capital(thing) + " is not working",
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " works again"}
}),
kindReloginNeeded: worded(func(o conditions.Observation) words {
module := ""
if o.Scope == conditions.ScopeModule {
module = idPart(o, 0)
}
return reloginWords(orModule(module), machineOr(o, "a machine"))
}),
kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
if consumer == "" {
consumer = "a module"
}
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
}),
kindProviderSilent: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: conditions.Capital(thing) + " went quiet",
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
"reporting.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " reports again"}
}),
kindRetireWaiting: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: "Retiring data on " + thing + " waits for you",
Needs: "approve or reject retiring it; the details list what would go.",
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
conditions.Capital(thing)),
Resolved: "Resolved: the retirement on " + thing + " is decided"}
}),
kindRetireRejected: worded(func(o conditions.Observation) words {
thing := conditions.ThingWords(o)
return words{Headline: "Unused data kept on " + thing,
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
Resolved: "Resolved: the unused data on " + thing + " is gone"}
}),
kindCleanupWaiting: worded(func(o conditions.Observation) words {
var where string
if o.Scope == conditions.ScopeProvider {
where = conditions.ThingWords(o)
} else {
what, _ := dataWords(o)
where = what + " on " + machineOr(o, "a machine")
}
return words{Headline: "Retired data waits for cleanup",
Needs: "delete it or keep it; nothing is deleted without your word.",
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
Resolved: "Resolved: the retired data is dealt with"}
}),
// Data.
kindDataUnmeasured: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Data on " + m + " could not be measured",
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
"the data there. The next measurement may settle it.", m),
Resolved: "Data on " + m + " is measured again"}
}),
kindProtectionMissing: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " is unprotected",
Needs: "check the storage it is on.",
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is protected again"}
}),
kindDataMissing: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " is gone",
Needs: "restore it from a backup, or silence this if you removed it.",
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is back", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}),
kindDataShrank: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
m := machineOr(o, "its machine")
if idPart(o, 1) == "dataset" {
what = "a dataset"
}
return words{Headline: conditions.Capital(what) + " shrank on " + m,
Needs: "if you meant it, silence this; if not, restore the last good copy from a backup.",
Explanation: "More than half of what it held is gone within a week.",
Resolved: "Resolved: the shrinking on " + m + " is explained", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
}),
kindDataQuiet: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
return words{Headline: conditions.Capital(what) + " stopped changing",
Needs: "check that what writes it still runs.",
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
Resolved: conditions.Capital(what) + " is written again"}
}),
kindBackupStale: worded(func(o conditions.Observation) words {
what, _ := dataWords(o)
m := machineOr(o, "its machine")
return words{Headline: "No recent backup of " + what,
Needs: fmt.Sprintf("check that backups run on %s.", m),
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
"changed since would be lost.", conditions.Capital(what), m),
Resolved: conditions.Capital(what) + " is backed up again"}
}),
kindArrayDegraded: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Storage on " + m + " is degraded",
Needs: fmt.Sprintf("check the disks of %s.", m),
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
Resolved: "Storage on " + m + " is whole again"}
}),
kindEmptyReplacement: worded(func(o conditions.Observation) words {
module := idPart(o, 1)
if o.Scope == conditions.ScopeProvider {
module = idPart(o, 2)
}
if module == "" {
module = "a module"
}
return words{Headline: conditions.Capital(module) + " runs on empty data",
Needs: "decide where its data lives: move the data, or move it back.",
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
Resolved: conditions.Capital(module) + " has its data again"}
}),
kindDataHeldTwice: worded(func(o conditions.Observation) words {
consumer := idPart(o, 1)
if consumer == "" {
consumer = "a module"
}
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
Needs: "decide which copy to keep.",
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
}),
// Deliveries and walks.
"stalled": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeDelivery {
return words{Headline: "A delivery is held too long",
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
Resolved: "Resolved: the delivery moves again"}
}
return words{Headline: "A delivery is stuck halfway",
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
Resolved: "Resolved: the delivery moves again"}
}),
kindWalkWaiting: worded(func(o conditions.Observation) words {
return words{Headline: "A delivery is waiting to start",
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
"out) has not let it start yet.",
Resolved: "Resolved: the delivery is no longer waiting"}
}),
"merge-not-acted": worded(func(o conditions.Observation) words {
return words{Headline: "A merge was never picked up",
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
"sent. The mesh catches up on missed merges by itself.",
Resolved: "Resolved: the merge was picked up"}
}),
"ask-lost": worded(func(o conditions.Observation) words {
return words{Headline: "A build has no result",
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
Resolved: "Resolved: the build has its result"}
}),
kindRolledBack: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
module := idPart(o, 0)
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
module = "a module"
}
if o.Scope == conditions.ScopeCore {
return words{Headline: "A core update was put back on " + m,
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
"before was put back and runs.", module, m),
Resolved: "Resolved: the core on " + m + " is healthy"}
}
return words{Headline: conditions.Capital(module) + " update put back on " + m,
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
"before, which runs. The change is not delivered until it is fixed.", module, m),
Resolved: "Resolved: " + module + " on " + m + " is settled"}
}),
kindRollbackFailed: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Putting back an update failed on " + m,
Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
Resolved: "Resolved: " + m + " runs a good build again"}
}),
"release-held": worded(func(o conditions.Observation) words {
return words{Headline: "Updates wait for your release",
Needs: "release them, or leave them held.",
Explanation: "Some module updates wait for a person to release them, and are not delivered until then.",
Resolved: "Resolved: the held updates are released"}
}),
"facts-stale": worded(func(o conditions.Observation) words {
return words{Headline: "Merge checks use outdated facts",
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
"by themselves.",
Resolved: "Merge checks use current facts again"}
}),
// The controller and the core.
"controller-deaf": worded(func(o conditions.Observation) words {
return words{Headline: "The controller stopped listening",
Needs: "restart the controller if this stays.",
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
"wait. Changes and repairs do not happen until it recovers.",
Resolved: "The controller listens again"}
}),
"self-check-silent": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh's self-check stopped",
Needs: "restart the controller if this stays.",
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
"Problems may go unnoticed until it runs again.",
Resolved: "The self-check runs again"}
}),
"watchdogs-silent": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh's watchdogs stopped",
Needs: "restart the controller if this stays.",
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
"missed signals are not noticed.",
Resolved: "The watchdogs run again"}
}),
"lease-lost": worded(func(o conditions.Observation) words {
return words{Headline: "The controller lost its lease",
Needs: "make sure only one controller runs; the details say which machines claim it.",
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
"not renew it, so two could act at once, or none.",
Resolved: "The controller holds its lease again"}
}),
"lease-split": worded(func(o conditions.Observation) words {
return words{Headline: "Two controllers may be acting",
Needs: "make sure only one controller runs; the details say which machines claim it.",
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
Resolved: "One controller acts again"}
}),
"stale-writer": worded(func(o conditions.Observation) words {
return words{Headline: "Outdated instructions are being sent",
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
"were refused, which is harmless.",
Resolved: "Resolved: no more outdated instructions"}
}),
"status-slow": worded(func(o conditions.Observation) words {
return words{Headline: "The mesh answers slowly",
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
Resolved: "The mesh answers quickly again"}
}),
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeBus {
return words{Headline: "The bus is not healthy",
Needs: "check the machine the bus runs on; the details say what fails.",
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
"and tool calls may fail until it recovers.",
Resolved: "The bus is healthy again"}
}
m := machineOr(o, "its machine")
return words{Headline: "The mesh's own software fails on " + m,
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
Resolved: "The mesh's own software on " + m + " is healthy again"}
}),
"call-hung": worded(func(o conditions.Observation) words {
return words{Headline: "A tool call is hanging",
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
Resolved: "Resolved: the tool call ended"}
}),
"probe-failed": worded(func(o conditions.Observation) words {
return words{Headline: "A check of the mesh could not run",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
"again every few minutes.",
Resolved: "The check runs again"}
}),
kindHealersBraked: worded(func(o conditions.Observation) words {
return words{Headline: "All automatic repairs stopped",
Needs: "read the details and decide whether the repairs may run again.",
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
"looks like a loop, so all of them stopped.",
Resolved: "Automatic repairs run again"}
}),
"healer-wanted": worded(func(o conditions.Observation) words {
cause := strings.TrimPrefix(o.ID, "hand-acts.")
if o.Scope != conditions.ScopeMesh {
cause = "something"
}
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
"broken now.",
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
}),
// The bus and seats.
"resolver-wrong": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "Name lookups fail on " + m,
Needs: fmt.Sprintf("check that %s is on and online.", m),
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
"machines that rely on it may not find each other.", m),
Resolved: "Name lookups work on " + m + " again"}
}),
"holder-silent": worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
seat := idPart(o, 0)
if o.Scope != conditions.ScopeSeat {
seat = "a seat"
}
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
"machine asleep is normal.", seat, m),
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
}),
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
return words{Headline: "The bus could not be fully set up",
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
"again with the next send.",
Resolved: "The bus is fully set up again"}
}),
kindBusMaintenance: worded(func(o conditions.Observation) words {
return words{Headline: "The bus is being upgraded",
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
Resolved: "The bus upgrade is done"}
}),
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
return words{Headline: "The bus upgrade failed",
Needs: "decide whether to put the bus back to the version before; the details say how.",
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
Resolved: "Resolved: the bus is healthy after its upgrade"}
}),
kindConsumerBehind: worded(func(o conditions.Observation) words {
return words{Headline: "Messages pile up for a listener",
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
"healer restarts it if it stays.",
Resolved: "The listener caught up"}
}),
"consumer-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is set up wrong",
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
"it up again.",
Resolved: "The listener is set up right again"}
}),
"consumer-lost": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is gone",
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
Resolved: "The listener is back"}
}),
"slow-consumer": worded(func(o conditions.Observation) words {
return words{Headline: "A listener on the bus is too slow",
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
Resolved: "The listener keeps up again"}
}),
"max-deliveries": worded(func(o conditions.Observation) words {
return words{Headline: "A message could not be handled",
Explanation: "The bus gave up on a message after trying to hand it over too many times.",
Resolved: "Resolved: messages are handled again"}
}),
"refused": worded(func(o conditions.Observation) words {
return words{Headline: "The bus refuses some messages",
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
Resolved: "Resolved: the bus takes the messages again"}
}),
"stream-wrong": worded(func(o conditions.Observation) words {
return words{Headline: "Part of the bus's storage is wrong",
Needs: "check the machine the bus runs on; the details say what is missing.",
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
Resolved: "The bus's storage is right again"}
}),
"archives-unheld": worded(func(o conditions.Observation) words {
return words{Headline: "Kept builds are about to be cleaned up",
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
Resolved: "Resolved: every kept build is claimed"}
}),
"archives-missing": worded(func(o conditions.Observation) words {
return words{Headline: "Kept builds are missing",
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
Resolved: "Resolved: the kept builds are back"}
}),
}
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
func humanDuration(d time.Duration) string {
switch {
case d < 2*time.Minute:
return "a minute"
case d < 2*time.Hour:
return fmt.Sprintf("%d minutes", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%d hours", int(d.Hours()))
}
return fmt.Sprintf("%d days", int(d.Hours()/24))
}
// repoName is a repository as a person says it: its name without its owner.
func repoName(repository string) string {
if i := strings.LastIndex(repository, "/"); i >= 0 {
return repository[i+1:]
}
return repository
}
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
func namesWords(names []string, most int) string {
switch {
case len(names) == 0:
return ""
case len(names) == 1:
return names[0]
case len(names) <= most:
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
}
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
}
// planModules are the modules a walk moves, sorted.
func planModules(p inventory.Plan) []string {
out := make([]string, 0, len(p.Modules))
for m := range p.Modules {
out = append(out, m)
}
sort.Strings(out)
return out
}
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
// their count and the repository.
func deliveryWhat(modules []string, repository string) string {
if len(modules) > 0 && len(modules) <= 2 {
return namesWords(modules, 2)
}
if len(modules) > 2 {
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
}
return repoName(repository)
}
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
func deliveryName(modules []string, repository string) string {
if len(modules) > 0 && len(modules) <= 2 {
return namesWords(modules, 2) + " delivery"
}
return repoName(repository) + " delivery"
}
// walkWaitingWords explains a walk waiting for its delivery's word.
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
what := deliveryWhat(w.modules, w.repository)
if severity == conditions.Urgent {
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
}
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
what, humanDuration(in), humanDuration(waitUrgentAfter))
}
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
func waitingNeeds(severity conditions.Severity) string {
if severity == conditions.Urgent {
return "start it, or stop it."
}
return ""
}
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
// the module that should have said go is the one not answering.
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
if severity != conditions.Urgent {
return nil
}
return []conditions.Action{
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
}
}
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
var actions []conditions.Action
for _, r := range rs {
if strings.Contains(r.Reason, "relogin needed") {
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
}
if r.Kind == link.KindUnit && len(actions) < 2 {
scope := "system"
if strings.Contains(r.Reason, "account's own") {
scope = "user"
}
label := "Restart"
if len(actions) > 0 {
label = "Restart " + unitPlainWords(r.Target)
}
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
}
}
if len(actions) > 0 {
return "restart it; if it fails again, the details say why.", actions
}
return "", nil
}
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
actions []conditions.Action) {
repository, _, _ := strings.Cut(l.ID, "@")
name := repoName(repository)
held := l.State
if held == "" {
held = "held"
}
long := "too long"
if d, err := time.ParseDuration(l.For); err == nil {
long = "for " + humanDuration(d)
}
if o.Resolver == conditions.ResolverOperator {
switch held {
case "held":
needs = "release it, or stop it."
actions = []conditions.Action{
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
}
case "ready", "checked":
needs = "merge its pull request, or close it."
default:
needs = "stop it, or read the details to see what it waits for."
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
}
}
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
}
// causeWords is a hand-act's cause as a person says it.
func causeWords(cause string) string {
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
}
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
func unitPlainWords(unit string) string {
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
if name, ok := strings.CutSuffix(unit, suffix); ok {
if what == "" {
return strings.ReplaceAll(name, ".", " ")
}
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
}
}
return strings.ReplaceAll(unit, ".", " ")
}
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
func resourcePlainWords(r inventory.ResourceHealth) string {
if r.Kind == link.KindUnit {
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
}
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
switch r.Reason {
case "restarting":
return what + " keeps restarting"
case "down":
return what + " is not running"
}
if r.Check != "" {
return what + " fails its " + r.Check + " check"
}
return what + " is not healthy"
}
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
// systemd's result, such as exit-code, or a sentence carrying it).
func unitReasonWords(reason string) string {
switch {
case strings.Contains(reason, "start-limit"):
return "failed too often to be started again"
case strings.Contains(reason, "exit-code"):
return "stopped with an error"
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
return "crashed"
case strings.Contains(reason, "timeout"):
return "timed out"
}
return "failed"
}
// orModule is a module's name, or "a module" when none is known.
func orModule(m string) string {
if m == "" {
return "a module"
}
return m
}
+186
View File
@@ -0,0 +1,186 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The conditions the operator read on 2026-10-08, raised again from the same facts (novox/hq ADR 0253):
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
// plainExample checks one finding's words as the keeper will keep them: plain, within their bounds,
// opened by the verdict, and saying what the operator needs; and its actions by label.
func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) {
t.Helper()
machines := append([]string{o.Machine}, o.Also...)
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
Actions: o.Actions}
if why, ok := conditions.PlainWords(w, machines...); !ok {
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
}
said := conditions.Verdict(o.Needs, o.Explanation)
if o.Headline != headline || said != explanation {
t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation)
}
var labels []string
for _, a := range o.Actions {
labels = append(labels, a.Label)
}
if strings.Join(labels, ",") != strings.Join(actions, ",") {
t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions)
}
quiet := ""
if o.Needs == "" && o.Severity == conditions.Warning {
quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)"
}
t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet,
said, labels, o.Resolved)
}
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no
// popup; past them, the operator starts or stops it from the notification.
func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) {
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
f := calm(now)
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}}
got := watchWaits(f)
if len(got) != 1 {
t.Fatalf("%+v", got)
}
plainExample(t, got[0], "openrazer delivery waiting to start",
"Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+
"for you if it still waits after 4 hours.")
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
}
// Past four hours it is urgent, and offers the controller's own answers.
f.waits[0].since = now.Add(-5 * time.Hour)
got = watchWaits(f)
plainExample(t, got[0], "openrazer delivery waiting to start",
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
"be stuck.", "Start", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
t.Errorf("start: %+v", a)
}
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
t.Errorf("stop: %+v", a)
}
// Many modules are counted, not listed in the headline.
f.waits[0].modules = []string{"a", "b", "c", "d"}
got = watchWaits(f)
if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") {
t.Errorf("%+v", got[0])
}
}
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
// account's own service manager (exit-code)". The operator restarts it from the notification.
func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
plainExample(t, o, "openrazer not working on g14",
"Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
t.Errorf("restart: %+v", a)
}
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
}
}
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …"
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
{Unit: "greenclip.service", Scope: "user", Result: "exit-code"}}})
if !raise {
t.Fatal("not raised")
}
plainExample(t, o, "3 failed services on shanks",
"Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+
"mesh does not repair them.", "Silence for a week")
}
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through
// the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup.
func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
f := calm(now)
for i := 0; i < 35; i++ {
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), "push")
a.Verb, a.By = "push", "g14.node-tools, through the mesh-controller seat"
f.handActs = append(f.handActs, a)
}
got := watchHandActs(f)
if len(got) != 1 {
t.Fatalf("%+v", got)
}
plainExample(t, got[0], "Push keeps being fixed by hand",
"Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+
"wanted for it. Nothing is broken now.")
}
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
plainExample(t, got[0], "Delivery of hq held for 36 hours",
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
"Release", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
t.Errorf("release: %+v", a)
}
}
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
// shape it is raised with.
func TestEveryWordingIsPlain(t *testing.T) {
subjects := []conditions.Observation{
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
{Scope: conditions.ScopeProvider, ID: "postgres.anchor.app_db", Machine: "anchor"},
{Scope: conditions.ScopePlan, ID: "plan-1791454185265004861"},
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
{Scope: conditions.ScopeBus, ID: "mesh"},
{Scope: conditions.ScopeSeat, ID: "node-backup.ace", Machine: "ace"},
{Scope: conditions.ScopeMesh, ID: "hand-acts.disk-load"},
{Scope: conditions.ScopeBuild, ID: "openrazer.g14", Machine: "g14"},
}
for kind, fn := range plainWordings {
if !conditions.Worded(kind) {
t.Errorf("%s is not registered", kind)
}
for _, s := range subjects {
s.Kind = kind
w := fn(s)
w.Explanation = conditions.Verdict(w.Needs, w.Explanation)
if why, ok := conditions.PlainWords(w, s.Machine); !ok {
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
}
}
}
}
+53
View File
@@ -29,7 +29,20 @@ func TestMain(m *testing.M) {
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
unsaid.note(o, field, r)
}
conditions.Unworded = func(o conditions.Observation, why string) {
if o.Source != "test" { // a kind a test makes up to exercise the keeper
unworded.add(o, why)
}
}
code := m.Run()
if said := unworded.all(); len(said) > 0 {
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+
"words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+
"reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n "))
if code == 0 {
code = 1
}
}
if said := unsaid.all(); len(said) > 0 {
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
@@ -74,6 +87,34 @@ func (u *unsayable) all() []string {
return out
}
// unworded collects, across the suite, every finding said in borrowed words (ADR 0253).
var unworded wordless
type wordless struct {
mu sync.Mutex
seen map[string]bool
}
func (u *wordless) add(o conditions.Observation, why string) {
u.mu.Lock()
defer u.mu.Unlock()
if u.seen == nil {
u.seen = map[string]bool{}
}
u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true
}
func (u *wordless) all() []string {
u.mu.Lock()
defer u.mu.Unlock()
var out []string
for s := range u.seen {
out = append(out, s)
}
sort.Strings(out)
return out
}
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
// that reads a producer's findings without raising them.
func linted(obs []conditions.Observation) []conditions.Observation {
@@ -85,6 +126,18 @@ func linted(obs []conditions.Observation) []conditions.Observation {
if r, ok := outward.Check(o.Summary, machines...); !ok {
unsaid.note(o, "summary", r)
}
switch {
case o.Headline != "":
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
if w.Resolved == "" {
w.Resolved = "Resolved"
}
if why, ok := conditions.PlainWords(w, machines...); !ok {
unworded.add(o, "its source's words are not plain: "+why)
}
case o.Kind != "" && !conditions.Worded(o.Kind):
unworded.add(o, "the kind "+o.Kind+" has no plain words")
}
}
return obs
}
+16 -2
View File
@@ -322,6 +322,10 @@ func watchPlans(f *signalFacts) []conditions.Observation {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway",
Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+
"usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))),
Resolved: "Delivery of " + repoName(p.repository) + " moves again",
Severity: conditions.Warning,
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
@@ -353,7 +357,12 @@ func watchWaits(f *signalFacts) []conditions.Observation {
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits)})
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: walkWaitingWords(w, in, severity),
Needs: waitingNeeds(severity),
Actions: waitingActions(w, severity),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
}
return out
}
@@ -681,7 +690,12 @@ func watchHandActs(f *signalFacts) []conditions.Observation {
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause,
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
Said: strings.Join(acts, "; ")})
Said: strings.Join(acts, "; "),
Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand",
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is "+
"wanted for it. Nothing is broken now.", causeWords(cause), repeated[cause],
int(handActsWithin.Hours()/24)),
Resolved: "Resolved: no more hand repairs of " + causeWords(cause)})
}
return out
}
+3 -1
View File
@@ -150,6 +150,8 @@ type planFacts struct {
type waitFacts struct {
id, repository, commit, awaits string
since time.Time
// modules are the modules the walk moves, as a person names the delivery.
modules []string
}
type loopFacts struct {
@@ -451,7 +453,7 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
// S16's, whatever mesh-delivery says or does not say.
if p.Waiting() {
waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit,
awaits: p.Delivery.Awaits, since: p.Created})
awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p)})
continue
}
_, paused := pausedWaiting(p, pause, now)
+19 -1
View File
@@ -125,8 +125,19 @@ type Condition struct {
Kind string `json:"kind"`
Subject Subject `json:"subject"`
Severity Severity `json:"severity"`
// Summary is one line in the mesh's words.
// Summary is one line in the mesh's words, for whoever looks closer: it may name plans, commits and
// the verbs that act.
Summary string `json:"summary"`
// Headline, Explanation and Resolved are what the operator reads, in plain words (novox/hq ADR 0253,
// plain.go): a few words naming the thing and what is wrong; one or two sentences on what happened,
// what it means and whether to act; the one line said when it clears.
Headline string `json:"headline"`
Explanation string `json:"explanation"`
Resolved string `json:"resolved"`
// Needs is what the operator does about it, one sentence; empty when nothing (the explanation then
// opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go).
Needs string `json:"needs"`
Actions []Action `json:"actions"`
// Evidence is the newest observations, at most KeptEvidence, newest first.
Evidence []Evidence `json:"evidence"`
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
@@ -175,6 +186,13 @@ type Observation struct {
Also []string
Severity Severity
Summary string
// Headline, Explanation and Resolved are the plain words the operator reads (plain.go). Left empty,
// the wording registered for Kind says them.
Headline string
Explanation string
Resolved string
Needs string
Actions []Action
// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
// here, never in Summary**: an address, a socket's error, a path or a name with its domain is
// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
+341
View File
@@ -0,0 +1,341 @@
package conditions
// Plain words (novox/hq ADR 0253): **a condition carries what the operator reads, in plain words, beside
// what an agent reads.** The summary is one line in the mesh's words for whoever looks closer — it names
// plans, commits and the verbs that act, and stays so. The operator reads a notification between other
// work: a popup that said "the walk of novox/mesh-catalog a6385479 has waited 56m0s for mesh-delivery's
// word to start: `mesh-delivery.show` …" was a wall of identifiers nobody could act on. So every condition
// also carries:
//
// - a **headline**: a few plain words naming the thing and what is wrong ("openrazer not running on
// g14"), the title of every message about it;
// - an **explanation**: one or two plain sentences — what happened, what it means for the operator,
// and whether they need to do anything;
// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14");
// - a **verdict**, which every explanation opens with: "Nothing for you to do." or "Needs you:" and one
// concrete thing the operator can do themselves (needs). Never "have an agent …": the operator is not
// asked to open a session to understand or answer a notification;
// - **actions**: what the operator may answer from the notification itself (Release, Stop, Restart,
// Silence), each the seat verb the operator channel calls when it is chosen, naming the operator.
//
// They are made here, where the condition is made, so every channel gets them — the desktop today,
// others later — and none has to guess what a key means. A producer may say them itself; otherwise the
// wording registered for its kind says them; otherwise a plain sentence made from its scope does, and a
// test suite is told (Unworded) so the kind gets words of its own. Each is held to the plain rule (Plain):
// no identifiers, hashes, keys, verb syntax, markup or clock times — a message names no time of its own,
// since the channel says when, in the operator's time.
import (
"fmt"
"regexp"
"strings"
"sync"
"github.com/novox/mesh-controller/internal/outward"
)
// Words are what the operator reads of a condition. Explanation is said after the verdict, which
// plainly writes from Needs: empty is "Nothing for you to do.", else "Needs you: " and Needs.
type Words struct {
Headline string
Explanation string
Resolved string
Needs string
Actions []Action
}
// Action is one answer the operator may give from a notification: a label, and the seat verb the
// operator channel calls with these arguments (and a why naming the operator and the label) when it is
// chosen. Machine is set for a seat every machine holds. An argument "why" given empty is the operator
// channel's to fill: it names the operator, the channel and the label chosen.
type Action struct {
Label string `json:"label"`
Verb string `json:"verb"`
Machine string `json:"machine,omitempty"`
Arguments map[string]string `json:"arguments,omitempty"`
}
// The two verdicts an explanation opens with.
const (
NothingToDo = "Nothing for you to do."
NeedsYou = "Needs you:"
)
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
// answer to a condition the operator decided to live with.
func SilenceAction(key string) Action {
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
}
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
const (
HeadlineMax = 60
ExplanationMax = 420
NeedsMax = 120
)
var (
wordingsMu sync.RWMutex
wordings = map[string]func(Observation) Words{}
)
// Wording registers the plain words of one condition kind.
func Wording(kind string, words func(Observation) Words) {
wordingsMu.Lock()
defer wordingsMu.Unlock()
wordings[kind] = words
}
// Worded says whether a kind has words of its own.
func Worded(kind string) bool {
wordingsMu.RLock()
defer wordingsMu.RUnlock()
_, ok := wordings[kind]
return ok
}
// Unworded is told of every observation said in borrowed words: its kind has none registered, or the
// words it was given break the plain rule. The keeper says it plainly anyway; a test suite sets this to
// fail the producer.
var Unworded func(o Observation, why string)
// The plain rule's shapes.
var (
hexID = regexp.MustCompile(`\b[0-9a-f]{7,40}\b`)
numberedID = regexp.MustCompile(`\b[a-z]+-[0-9]{6,}\b`)
dotted = regexp.MustCompile(`\b[a-z][a-z0-9_-]*\.[a-z][a-z0-9_-]*\b`)
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
)
// Plain says whether text is plain words the operator reads at a glance, and if not, what is not: an
// identifier (a hash, a numbered id, a dotted name such as a key or a verb), a Go duration, a clock time
// or date (the channel says when, in the operator's time), a command's flag, markup, a line break, or
// anything the operator's channel withholds (ADR 0234 §6). machines are names that may appear.
func Plain(text string, machines ...string) (string, bool) {
if strings.TrimSpace(text) == "" {
return "nothing is said", false
}
if strings.ContainsAny(text, "\n\r\t") {
return "a line break", false
}
if m := markup.FindString(text); m != "" {
return "markup (" + m + ")", false
}
for _, shape := range []struct {
re *regexp.Regexp
what string
}{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"},
{clockTime, "a clock time or date"}, {flag, "a command's flag"}} {
if m := shape.re.FindString(text); m != "" {
if shape.re == hexID && !strings.ContainsAny(m, "0123456789") {
continue // a word of letters a to f only
}
return shape.what + " (" + strings.TrimSpace(m) + ")", false
}
}
if r, ok := outward.Check(text, machines...); !ok {
return r.String(), false
}
return "", true
}
// PlainWords checks all three, with their bounds.
func PlainWords(w Words, machines ...string) (string, bool) {
if why, ok := Plain(w.Headline, machines...); !ok {
return "headline: " + why, false
}
if len(w.Headline) > HeadlineMax {
return fmt.Sprintf("headline: longer than %d characters", HeadlineMax), false
}
if why, ok := Plain(w.Explanation, machines...); !ok {
return "explanation: " + why, false
}
if len(w.Explanation) > ExplanationMax {
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
}
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
return "explanation: sends the operator elsewhere (" + m + ")", false
}
}
if w.Needs != "" {
if why, ok := Plain(w.Needs, machines...); !ok {
return "needs: " + why, false
}
if m := agentWord.FindString(w.Needs); m != "" {
return "needs: not something the operator does themselves (" + m + ")", false
}
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
return fmt.Sprintf("needs: one sentence of at most %d characters, ending in a full stop", NeedsMax), false
}
}
for _, a := range w.Actions {
if a.Label == "" || len(a.Label) > 24 || a.Verb == "" {
return fmt.Sprintf("action %q: a short label and a verb", a.Label), false
}
}
if why, ok := Plain(w.Resolved, machines...); !ok {
return "resolved: " + why, false
}
if len(w.Resolved) > HeadlineMax+20 {
return fmt.Sprintf("resolved: longer than %d characters", HeadlineMax+20), false
}
return "", true
}
// plainly gives an observation its plain words: its own, its kind's, or its scope's.
func plainly(o Observation) Observation {
machines := append([]string{o.Machine}, o.Also...)
given := Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
Actions: o.Actions}
var w Words
from := ""
switch {
case given.Headline != "":
w, from = given, "its source"
default:
wordingsMu.RLock()
fn := wordings[o.Kind]
wordingsMu.RUnlock()
if fn != nil {
w, from = fn(o), "the wording of "+o.Kind
if given.Explanation != "" {
w.Explanation = given.Explanation
}
if given.Resolved != "" {
w.Resolved = given.Resolved
}
if given.Needs != "" {
w.Needs = given.Needs
}
if len(given.Actions) > 0 {
w.Actions = given.Actions
}
}
}
if w.Headline != "" && w.Resolved == "" {
w.Resolved = "Resolved: " + lowerFirst(w.Headline)
}
if from == "" {
if Unworded != nil {
Unworded(o, "the kind "+o.Kind+" has no plain words")
}
w = scopeWords(o)
} else if why, ok := PlainWords(w, machines...); !ok {
if Unworded != nil {
Unworded(o, from+" is not plain: "+why)
}
w = scopeWords(o)
}
o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
o.Explanation = Verdict(w.Needs, w.Explanation)
return o
}
// Verdict is an explanation opened by its verdict.
func Verdict(needs, explanation string) string {
if needs == "" {
return strings.TrimSpace(NothingToDo + " " + explanation)
}
return strings.TrimSpace(NeedsYou + " " + needs + " " + explanation)
}
// Escalated is what a condition needs once a healer gave up on it, where its words needed nothing.
const Escalated = "the mesh tried to repair this and could not; read the details to decide what to do."
// escalatedWords turns "Nothing for you to do." into "Needs you:" once a healer gave up.
func escalatedWords(c *Condition) {
if c.Escalated() && c.Needs == "" {
c.Needs = Escalated
c.Explanation = Verdict(c.Needs, strings.TrimSpace(strings.TrimPrefix(c.Explanation, NothingToDo)))
}
}
// scopeWords is what is said of a kind with no words of its own: what kind of thing, and which one
// where its name is a name (a machine, a module), never an id.
func scopeWords(o Observation) Words {
thing := ThingWords(o)
what := strings.ReplaceAll(o.Kind, "-", " ")
w := Words{
Headline: Capital(thing) + " needs a look",
Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s.", what, thing),
Resolved: "Resolved: " + thing + " is fine again",
Needs: ResolverNeeds(o),
}
if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok {
w = Words{Headline: "Something in the mesh needs a look",
Explanation: "The mesh noticed a problem it has no plain words for yet.",
Resolved: "Resolved: the mesh is fine again", Needs: ResolverNeeds(o)}
}
return w
}
// ThingWords is what a condition is about, in plain words: "the machine ace", "openrazer on g14", "a
// delivery". An id that is not a name (a plan's, a delivery's, a call's) is never said.
func ThingWords(o Observation) string {
switch o.Scope {
case ScopeMachine:
if o.Machine != "" {
return o.Machine
}
return "a machine"
case ScopeModule:
module := strings.TrimSuffix(o.ID, "."+o.Machine)
if o.Machine != "" && module != "" && !strings.Contains(module, ".") {
return module + " on " + o.Machine
}
return "a module"
case ScopeProvider:
if module, _, _ := strings.Cut(o.ID, "."); module != "" && o.Machine != "" {
return module + " on " + o.Machine
}
return "a provider"
case ScopePlan:
return "a walk"
case ScopeDelivery:
return "a delivery"
case ScopeBuild:
return "a build"
case ScopeCall:
return "a tool call"
case ScopeMerge:
return "a merge"
case ScopeSeat:
return "a seat's holder"
case ScopeBus:
return "the bus"
case ScopeCore:
return "the controller"
case ScopeProbe:
return "the self-check"
}
return "the mesh"
}
// ResolverNeeds is what the operator needs to do, from who resolves it: nothing, unless only a person can.
func ResolverNeeds(o Observation) string {
if o.Resolver == ResolverOperator {
return "the mesh does not repair this by itself; read the details to decide what to do."
}
return ""
}
// Capital is s with its first letter upper case.
func Capital(s string) string {
if s == "" {
return s
}
return strings.ToUpper(s[:1]) + s[1:]
}
func lowerFirst(s string) string {
if s == "" || (len(s) > 1 && strings.ToUpper(s[:2]) == s[:2]) {
return s
}
return strings.ToLower(s[:1]) + s[1:]
}
+163
View File
@@ -0,0 +1,163 @@
package conditions
import (
"encoding/json"
"strings"
"testing"
)
// **The plain rule** (novox/hq ADR 0253): what the operator reads names no identifier, no code and no time
// of its own; the words conditions are made of pass.
func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) {
refused := map[string]string{
"the walk of novox/mesh-catalog a6385479 waits": "a hash",
"plan plan-1791454185265004861 is waiting": "a hash",
"see `mesh-delivery.show` for why": "markup",
"call mesh-delivery.show for why": "a dotted name",
"key plan.x.waiting": "a dotted name",
"waited 56m0s": "a duration in code",
"since 10:40": "a clock time",
"since 2026-10-08": "a clock time",
"raised at noon UTC": "a clock time",
"start it with plans go --why": "a command's flag",
"one line\nand another": "a line break",
"the resolver at 10.77.0.1 is silent": "",
"": "nothing",
"**bold**": "markup",
"storage-media.mount failed": "a dotted name",
"the store at /var/lib/mesh is full": "",
"a commit 0123abc landed": "a hash",
}
for text, want := range refused {
why, ok := Plain(text, "g14")
if ok {
t.Errorf("%q passed", text)
continue
}
if want != "" && !strings.HasPrefix(why, want) {
t.Errorf("%q: refused for %q, want %q", text, why, want)
}
}
for _, text := range []string{
"openrazer delivery waiting to start",
"The change to openrazer is merged and built, and has waited 56 minutes for mesh-delivery to let it start.",
"3 failed services on shanks",
"On shanks, mnt-recalbox (a mount), storage-media (a mount) and greenclip failed.",
"Name lookups fail on g14",
"Retired data of postgres on anchor waits for app_db",
"Push keeps being fixed by hand",
} {
if why, ok := Plain(text, "g14", "shanks", "anchor"); !ok {
t.Errorf("%q refused: %s", text, why)
}
}
}
// **A condition carries its plain words**: a producer's own; else its kind's; else its scope's — and the
// event says them beside the summary, for every channel.
func TestAConditionCarriesItsPlainWords(t *testing.T) {
k, _, told, _ := keeper(t)
before := Unworded
var unworded []string
Unworded = func(o Observation, why string) { unworded = append(unworded, why) }
t.Cleanup(func() { Unworded = before })
Wording("test-silent", func(o Observation) Words {
return Words{Headline: o.Machine + " is not answering", Explanation: "The mesh has not heard from it.",
Resolved: o.Machine + " answers again"}
})
c, err := k.Observe(t.Context(), Observation{Scope: ScopeMachine, ID: "ace", Kind: "test-silent", Machine: "ace",
Severity: Warning, Source: "S1", Summary: "ace has not been heard from since 12:00 UTC (bound 3m0s)"})
if err != nil {
t.Fatal(err)
}
if c.Headline != "ace is not answering" || c.Resolved != "ace answers again" || c.Explanation == "" {
t.Fatalf("the kind's words were not kept: %+v", c)
}
if c.Summary != "ace has not been heard from since 12:00 UTC (bound 3m0s)" {
t.Errorf("the summary was changed: %q", c.Summary)
}
said := settled(t, told, 1)
body, _ := json.Marshal(said[0])
for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"Nothing for you to do. The mesh has not heard from it."`,
`"resolved":"ace answers again"`} {
if !strings.Contains(string(body), field) {
t.Errorf("the event does not carry %s: %s", field, body)
}
}
// A producer's own words win; a resolved line it leaves out is made from its headline.
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Kind: "test-silent",
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy",
Headline: "openrazer not working on g14", Explanation: "Its service stopped with an error."})
if c.Headline != "openrazer not working on g14" || c.Resolved != "Resolved: openrazer not working on g14" {
t.Errorf("the producer's words: %+v", c)
}
if len(unworded) != 0 {
t.Errorf("words that were plain were reported: %v", unworded)
}
// Words that are not plain, and a kind with none, are said from the scope — and reported.
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded",
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"})
if c.Headline != "Openrazer on g14 needs a look" || !strings.HasPrefix(c.Explanation, NothingToDo) {
t.Errorf("the scope's words: %+v", c)
}
c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent",
Severity: Warning, Source: "S16", Summary: "the walk waits", Headline: "plan-1791454185265004861 waits"})
if strings.Contains(c.Headline, "plan-") || c.Headline != "A walk needs a look" {
t.Errorf("an id reached the headline: %+v", c)
}
if len(unworded) != 2 || !strings.Contains(unworded[0], "no plain words") || !strings.Contains(unworded[1], "not plain") {
t.Errorf("what was said in borrowed words was not reported: %v", unworded)
}
}
// **An explanation opens with its verdict, and never sends the operator to an agent.**
func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
if got := Verdict("", "It may be asleep."); got != "Nothing for you to do. It may be asleep." {
t.Errorf("%q", got)
}
if got := Verdict("release it, or stop it.", "It is held."); got != "Needs you: release it, or stop it. It is held." {
t.Errorf("%q", got)
}
for _, w := range []Words{
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
} {
if _, ok := PlainWords(w); ok {
t.Errorf("passed: %+v", w)
}
}
if why, ok := PlainWords(Words{Headline: "push keeps being fixed by hand", Explanation: "A person repaired " +
"push by hand 35 times.", Resolved: "Resolved", Needs: "release it, or stop it.",
Actions: []Action{{Label: "Release", Verb: "mesh-delivery.release"}}}); !ok {
t.Errorf("refused: %s", why)
}
}
// **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the
// budget is spent, the operator is needed.
func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
k, _, _, _ := keeper(t)
ctx := t.Context()
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
if _, err := k.Observe(ctx, o); err != nil {
t.Fatal(err)
}
c, _, err := k.Escalate(ctx, o.Key(), Attempt{What: "budget spent", Outcome: "escalated", By: "healer H1"})
if err != nil {
t.Fatal(err)
}
if want := "Needs you: " + Escalated + " It may be asleep."; c.Explanation != want || c.Needs != Escalated {
t.Fatalf("escalated: %q, want %q", c.Explanation, want)
}
c, _ = k.Observe(ctx, o)
if strings.Count(c.Explanation, Escalated) != 1 || strings.Contains(c.Explanation, NothingToDo) {
t.Errorf("seen again after escalation: %q", c.Explanation)
}
}
+6 -1
View File
@@ -175,6 +175,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
return Condition{}, err
}
o = k.sayable(o)
o = plainly(o)
key := o.Key()
for i := 0; i < tries; i++ {
now := k.now().UTC()
@@ -188,7 +189,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
}
if !found {
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}},
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
Resolver: orSelf(o.Resolver)}
change := ChangeRaised
@@ -238,6 +240,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
// The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
escalatedWords(&c)
if o.Machine != "" {
c.Subject.Machine = o.Machine
}
@@ -335,6 +339,7 @@ func (k *Keeper) Escalate(ctx context.Context, key string, a Attempt) (Condition
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
c.Resolver = ResolverOperator
}
escalatedWords(c)
return changes
})
}