diff --git a/cmd/mesh-controller/delivery_conditions.go b/cmd/mesh-controller/delivery_conditions.go index 439caf80..b8f4cbd0 100644 --- a/cmd/mesh-controller/delivery_conditions.go +++ b/cmd/mesh-controller/delivery_conditions.go @@ -113,6 +113,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation { if l.operatorsOnly() { o.Resolver = conditions.ResolverOperator } + o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o) out = append(out, o) } return out diff --git a/cmd/mesh-controller/machine_units.go b/cmd/mesh-controller/machine_units.go index 2f51b57b..0060f0fb 100644 --- a/cmd/mesh-controller/machine_units.go +++ b/cmd/mesh-controller/machine_units.go @@ -90,8 +90,9 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser if len(u.Failed) == 0 { return o, false } - var names, said []string + var names, said, plain []string for _, f := range u.Failed { + plain = append(plain, unitPlainWords(f.Unit)) where := "" if f.Scope == "user" { where = " (the account's own manager)" @@ -111,6 +112,16 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser "Each is the machine's own: mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")) o.Said = strings.Join(said, "; ") + failed := "1 failed service" + if len(u.Failed) > 1 { + failed = fmt.Sprintf("%d failed services", len(u.Failed)) + } + o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node) + o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node) + o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.", + node, namesWords(plain, 3)) + o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())} + o.Resolved = "No failed services on " + node + " any more" return o, true } diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 25e75e05..d402c94a 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -167,6 +167,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi o.Severity = conditions.Urgent o.Said += "; " + waitingWords(waiters) o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) + o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters)) } } seen[o.Key()] = true @@ -241,8 +242,10 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module, // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence. func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { - var words, said []string + var words, said, plain []string + needs, actions := moduleNeeds(node, rs) for _, r := range rs { + plain = append(plain, resourcePlainWords(r)) if r.Kind == link.KindUnit { // A failed unit is named by the unit, which is what a person looks for (issue 315); the // resource that places it — a package, a file — is evidence. @@ -258,8 +261,14 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal } return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth, - Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")), - Said: strings.Join(said, "; ")} + Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")), + Said: strings.Join(said, "; "), + Headline: fmt.Sprintf("%s not working on %s", module, node), + Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node, + namesWords(plain, 3)), + Needs: needs, + Actions: actions, + Resolved: fmt.Sprintf("%s works again on %s", module, node)} } // reasonWords is why a resource is unhealthy, as a person reads it. diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go new file mode 100644 index 00000000..edd6a849 --- /dev/null +++ b/cmd/mesh-controller/plain_words.go @@ -0,0 +1,761 @@ +package main + +// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator +// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it +// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer. +// +// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's +// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test +// suite is held to having words, and to their being plain (plain_words_test.go). + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +type words = conditions.Words + +// worded is a wording that needs only the condition's subject. +func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn } + +func init() { + for kind, fn := range plainWordings { + conditions.Wording(kind, fn) + } +} + +// machineOr is the machine a condition concerns, or what to say when none is named. +func machineOr(o conditions.Observation, none string) string { + if o.Machine != "" { + return o.Machine + } + return none +} + +// idPart is the n-th dotted part of a condition's id, or "". +func idPart(o conditions.Observation, n int) string { + parts := strings.Split(o.ID, ".") + if n < len(parts) { + return parts[n] + } + return "" +} + +// dataWords is " of " for a data condition keyed by machine, module and item. +func dataWords(o conditions.Observation) (what, module string) { + module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ") + switch { + case module != "" && item != "": + return item + " of " + module, module + case module != "": + return "data of " + module, module + } + return "data", "a module" +} + +var plainWordings = map[string]func(conditions.Observation) words{ + // Machines and their node-engine. + "silent": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + w := words{Headline: m + " is not answering", + Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+ + "normal.", m), + Resolved: m + " answers again"} + if o.Severity == conditions.Urgent { + w.Needs = fmt.Sprintf("check that %s is on and online.", m) + w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+ + "nothing changes anywhere until it is back.", m) + } + return w + }), + "sent-not-reported": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " has not confirmed a change", + Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+ + "It usually catches up by itself.", m, m), + Resolved: m + " applied the change"} + }), + "tools-silent": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "Tools on " + m + " are not answering", + Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+ + "the mesh until it does. A machine asleep is normal.", m, m), + Resolved: "Tools on " + m + " answer again"} + }), + kindAwaitingPush: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " waits for new instructions", + Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.", + conditions.Capital(m)), + Resolved: m + " got its new instructions"} + }), + "declaration-refused": worded(func(o conditions.Observation) words { + if o.Scope == conditions.ScopeMesh { + return words{Headline: "No machine can get new instructions", + Needs: "decide whether to undo the last catalogue change; the details say what fails.", + Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " + + "made. Machines keep running what they have.", + Resolved: "Machines can get new instructions again"} + } + m := machineOr(o, "a machine") + return words{Headline: m + " cannot get new instructions", + Needs: "decide whether to undo the last catalogue change; the details say what fails.", + Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+ + "reaches it. It keeps running what it has.", m), + Resolved: m + " can get new instructions again"} + }), + "own-address-banned": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " has banned the mesh", + Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m), + Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+ + "mesh is locked out of it.", m), + Resolved: m + " no longer bans the mesh"} + }), + "core-behind": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " runs an old node-engine", + Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+ + "works, and gets the newer one with a later delivery.", conditions.Capital(m)), + Resolved: m + " runs the current node-engine"} + }), + kindMachineUnits: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "Failed services on " + m, + Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m), + Explanation: "No module manages them, so the mesh does not repair them.", + Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}} + }), + kindMachineNetwork: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " has a network problem", + Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+ + "route. It often passes again once a network change settles.", conditions.Capital(m)), + Resolved: m + "'s network is fine again"} + }), + kindNetworkRewritten: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + "'s network settings were changed", + Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+ + "client. Mesh names may not resolve there until it is undone.", m), + Resolved: m + "'s network settings are the mesh's again"} + }), + kindNetworkUnreachable: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " cannot be reached", + Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+ + "asleep is normal.", m), + Resolved: m + " can be reached again"} + }), + kindBindingKept: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "A module's data source is held on " + m, + Needs: "decide where its data lives; the details name both places.", + Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+ + "kept the old one because switching would leave the data behind.", m), + Resolved: "Resolved: the data source on " + m + " is decided"} + }), + kindBindingMoving: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "A module's data source moves on " + m, + Needs: "move its data to the new provider before the next push.", + Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m), + Resolved: "Resolved: the data source move on " + m + " is settled"} + }), + kindBindingMoved: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "A module's data source moved on " + m, + Needs: "check that it found its data.", + Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m), + Resolved: "Resolved: the data source move on " + m + " is settled"} + }), + + // Modules and providers. + kindModuleUnhealthy: worded(func(o conditions.Observation) words { + thing := conditions.ThingWords(o) + return words{Headline: conditions.Capital(thing) + " is not working", + Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)), + Resolved: conditions.Capital(thing) + " works again"} + }), + kindProviderFailing: worded(func(o conditions.Observation) words { + thing, consumer := conditions.ThingWords(o), idPart(o, 2) + if consumer == "" { + consumer = "a module" + } + return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer, + Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+ + "it up. It tries again by itself.", conditions.Capital(thing), consumer), + Resolved: conditions.Capital(thing) + " serves " + consumer + " again"} + }), + kindProviderSilent: worded(func(o conditions.Observation) words { + thing := conditions.ThingWords(o) + return words{Headline: conditions.Capital(thing) + " went quiet", + Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+ + "reporting.", conditions.Capital(thing)), + Resolved: conditions.Capital(thing) + " reports again"} + }), + kindRetireWaiting: worded(func(o conditions.Observation) words { + thing := conditions.ThingWords(o) + return words{Headline: "Retiring data on " + thing + " waits for you", + Needs: "approve or reject retiring it; the details list what would go.", + Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.", + conditions.Capital(thing)), + Resolved: "Resolved: the retirement on " + thing + " is decided"} + }), + kindRetireRejected: worded(func(o conditions.Observation) words { + thing := conditions.ThingWords(o) + return words{Headline: "Unused data kept on " + thing, + Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing), + Resolved: "Resolved: the unused data on " + thing + " is gone"} + }), + kindCleanupWaiting: worded(func(o conditions.Observation) words { + var where string + if o.Scope == conditions.ScopeProvider { + where = conditions.ThingWords(o) + } else { + what, _ := dataWords(o) + where = what + " on " + machineOr(o, "a machine") + } + return words{Headline: "Retired data waits for cleanup", + Needs: "delete it or keep it; nothing is deleted without your word.", + Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where), + Resolved: "Resolved: the retired data is dealt with"} + }), + + // Data. + kindDataUnmeasured: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "Data on " + m + " could not be measured", + Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+ + "the data there. The next measurement may settle it.", m), + Resolved: "Data on " + m + " is measured again"} + }), + kindProtectionMissing: worded(func(o conditions.Observation) words { + what, _ := dataWords(o) + return words{Headline: conditions.Capital(what) + " is unprotected", + Needs: "check the storage it is on.", + Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+ + "the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")), + Resolved: conditions.Capital(what) + " is protected again"} + }), + kindDataMissing: worded(func(o conditions.Observation) words { + what, _ := dataWords(o) + return words{Headline: conditions.Capital(what) + " is gone", + Needs: "restore it from a backup, or silence this if you removed it.", + Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")), + Resolved: conditions.Capital(what) + " is back", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}} + }), + kindDataShrank: worded(func(o conditions.Observation) words { + what, _ := dataWords(o) + m := machineOr(o, "its machine") + if idPart(o, 1) == "dataset" { + what = "a dataset" + } + return words{Headline: conditions.Capital(what) + " shrank on " + m, + Needs: "if you meant it, silence this; if not, restore the last good copy from a backup.", + Explanation: "More than half of what it held is gone within a week.", + Resolved: "Resolved: the shrinking on " + m + " is explained", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}} + }), + kindDataQuiet: worded(func(o conditions.Observation) words { + what, _ := dataWords(o) + return words{Headline: conditions.Capital(what) + " stopped changing", + Needs: "check that what writes it still runs.", + Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+ + "usual.", conditions.Capital(what), machineOr(o, "its machine")), + Resolved: conditions.Capital(what) + " is written again"} + }), + kindBackupStale: worded(func(o conditions.Observation) words { + what, _ := dataWords(o) + m := machineOr(o, "its machine") + return words{Headline: "No recent backup of " + what, + Needs: fmt.Sprintf("check that backups run on %s.", m), + Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+ + "changed since would be lost.", conditions.Capital(what), m), + Resolved: conditions.Capital(what) + " is backed up again"} + }), + kindArrayDegraded: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "Storage on " + m + " is degraded", + Needs: fmt.Sprintf("check the disks of %s.", m), + Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.", + Resolved: "Storage on " + m + " is whole again"} + }), + kindEmptyReplacement: worded(func(o conditions.Observation) words { + module := idPart(o, 1) + if o.Scope == conditions.ScopeProvider { + module = idPart(o, 2) + } + if module == "" { + module = "a module" + } + return words{Headline: conditions.Capital(module) + " runs on empty data", + Needs: "decide where its data lives: move the data, or move it back.", + Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+ + "elsewhere.", conditions.Capital(module), machineOr(o, "its machine")), + Resolved: conditions.Capital(module) + " has its data again"} + }), + kindDataHeldTwice: worded(func(o conditions.Observation) words { + consumer := idPart(o, 1) + if consumer == "" { + consumer = "a module" + } + return words{Headline: conditions.Capital(consumer) + "'s data is on several machines", + Needs: "decide which copy to keep.", + Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)), + Resolved: conditions.Capital(consumer) + "'s data is in one place again"} + }), + + // Deliveries and walks. + "stalled": worded(func(o conditions.Observation) words { + if o.Scope == conditions.ScopeDelivery { + return words{Headline: "A delivery is held too long", + Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o), + Resolved: "Resolved: the delivery moves again"} + } + return words{Headline: "A delivery is stuck halfway", + Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.", + Resolved: "Resolved: the delivery moves again"} + }), + kindWalkWaiting: worded(func(o conditions.Observation) words { + return words{Headline: "A delivery is waiting to start", + Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " + + "out) has not let it start yet.", + Resolved: "Resolved: the delivery is no longer waiting"} + }), + "merge-not-acted": worded(func(o conditions.Observation) words { + return words{Headline: "A merge was never picked up", + Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " + + "sent. The mesh catches up on missed merges by itself.", + Resolved: "Resolved: the merge was picked up"} + }), + "ask-lost": worded(func(o conditions.Observation) words { + return words{Headline: "A build has no result", + Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", + Resolved: "Resolved: the build has its result"} + }), + kindRolledBack: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + module := idPart(o, 0) + if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore { + module = "a module" + } + if o.Scope == conditions.ScopeCore { + return words{Headline: "A core update was put back on " + m, + Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+ + "before was put back and runs.", module, m), + Resolved: "Resolved: the core on " + m + " is healthy"} + } + return words{Headline: conditions.Capital(module) + " update put back on " + m, + Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+ + "before, which runs. The change is not delivered until it is fixed.", module, m), + Resolved: "Resolved: " + module + " on " + m + " is settled"} + }), + kindRollbackFailed: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "Putting back an update failed on " + m, + Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m), + Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m), + Resolved: "Resolved: " + m + " runs a good build again"} + }), + "release-held": worded(func(o conditions.Observation) words { + return words{Headline: "Updates wait for your release", + Needs: "release them, or leave them held.", + Explanation: "Some module updates wait for a person to release them, and are not delivered until then.", + Resolved: "Resolved: the held updates are released"} + }), + "facts-stale": worded(func(o conditions.Observation) words { + return words{Headline: "Merge checks use outdated facts", + Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " + + "by themselves.", + Resolved: "Merge checks use current facts again"} + }), + + // The controller and the core. + "controller-deaf": worded(func(o conditions.Observation) words { + return words{Headline: "The controller stopped listening", + Needs: "restart the controller if this stays.", + Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " + + "wait. Changes and repairs do not happen until it recovers.", + Resolved: "The controller listens again"} + }), + "self-check-silent": worded(func(o conditions.Observation) words { + return words{Headline: "The mesh's self-check stopped", + Needs: "restart the controller if this stays.", + Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " + + "Problems may go unnoticed until it runs again.", + Resolved: "The self-check runs again"} + }), + "watchdogs-silent": worded(func(o conditions.Observation) words { + return words{Headline: "The mesh's watchdogs stopped", + Needs: "restart the controller if this stays.", + Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " + + "missed signals are not noticed.", + Resolved: "The watchdogs run again"} + }), + "lease-lost": worded(func(o conditions.Observation) words { + return words{Headline: "The controller lost its lease", + Needs: "make sure only one controller runs; the details say which machines claim it.", + Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " + + "not renew it, so two could act at once, or none.", + Resolved: "The controller holds its lease again"} + }), + "lease-split": worded(func(o conditions.Observation) words { + return words{Headline: "Two controllers may be acting", + Needs: "make sure only one controller runs; the details say which machines claim it.", + Explanation: "The lease that lets one controller act at a time does not agree with its record.", + Resolved: "One controller acts again"} + }), + "stale-writer": worded(func(o conditions.Observation) words { + return words{Headline: "Outdated instructions are being sent", + Explanation: "Something kept sending instructions older than what the machines already hold, and they " + + "were refused, which is harmless.", + Resolved: "Resolved: no more outdated instructions"} + }), + "status-slow": worded(func(o conditions.Observation) words { + return words{Headline: "The mesh answers slowly", + Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.", + Resolved: "The mesh answers quickly again"} + }), + kindCoreUnhealthy: worded(func(o conditions.Observation) words { + if o.Scope == conditions.ScopeBus { + return words{Headline: "The bus is not healthy", + Needs: "check the machine the bus runs on; the details say what fails.", + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " + + "and tool calls may fail until it recovers.", + Resolved: "The bus is healthy again"} + } + m := machineOr(o, "its machine") + return words{Headline: "The mesh's own software fails on " + m, + Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m), + Resolved: "The mesh's own software on " + m + " is healthy again"} + }), + "call-hung": worded(func(o conditions.Observation) words { + return words{Headline: "A tool call is hanging", + Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.", + Resolved: "Resolved: the tool call ended"} + }), + "probe-failed": worded(func(o conditions.Observation) words { + return words{Headline: "A check of the mesh could not run", + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " + + "again every few minutes.", + Resolved: "The check runs again"} + }), + kindHealersBraked: worded(func(o conditions.Observation) words { + return words{Headline: "All automatic repairs stopped", + Needs: "read the details and decide whether the repairs may run again.", + Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " + + "looks like a loop, so all of them stopped.", + Resolved: "Automatic repairs run again"} + }), + "healer-wanted": worded(func(o conditions.Observation) words { + cause := strings.TrimPrefix(o.ID, "hand-acts.") + if o.Scope != conditions.ScopeMesh { + cause = "something" + } + return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand", + Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " + + "broken now.", + Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")} + }), + + // The bus and seats. + "resolver-wrong": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "Name lookups fail on " + m, + Needs: fmt.Sprintf("check that %s is on and online.", m), + Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+ + "machines that rely on it may not find each other.", m), + Resolved: "Name lookups work on " + m + " again"} + }), + "holder-silent": worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + seat := idPart(o, 0) + if o.Scope != conditions.ScopeSeat { + seat = "a seat" + } + return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering", + Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+ + "machine asleep is normal.", seat, m), + Resolved: conditions.Capital(seat) + " on " + m + " answers again"} + }), + kindBusObjectsUnasserted: worded(func(o conditions.Observation) words { + return words{Headline: "The bus could not be fully set up", + Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " + + "again with the next send.", + Resolved: "The bus is fully set up again"} + }), + kindBusMaintenance: worded(func(o conditions.Observation) words { + return words{Headline: "The bus is being upgraded", + Explanation: "The mesh's message system is being upgraded; some things pause until it is done.", + Resolved: "The bus upgrade is done"} + }), + kindBusUpgradeFailed: worded(func(o conditions.Observation) words { + return words{Headline: "The bus upgrade failed", + Needs: "decide whether to put the bus back to the version before; the details say how.", + Explanation: "The upgrade of the mesh's message system did not end healthy in time.", + Resolved: "Resolved: the bus is healthy after its upgrade"} + }), + kindConsumerBehind: worded(func(o conditions.Observation) words { + return words{Headline: "Messages pile up for a listener", + Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " + + "healer restarts it if it stays.", + Resolved: "The listener caught up"} + }), + "consumer-wrong": worded(func(o conditions.Observation) words { + return words{Headline: "A listener on the bus is set up wrong", + Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " + + "it up again.", + Resolved: "The listener is set up right again"} + }), + "consumer-lost": worded(func(o conditions.Observation) words { + return words{Headline: "A listener on the bus is gone", + Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", + Resolved: "The listener is back"} + }), + "slow-consumer": worded(func(o conditions.Observation) words { + return words{Headline: "A listener on the bus is too slow", + Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.", + Resolved: "The listener keeps up again"} + }), + "max-deliveries": worded(func(o conditions.Observation) words { + return words{Headline: "A message could not be handled", + Explanation: "The bus gave up on a message after trying to hand it over too many times.", + Resolved: "Resolved: messages are handled again"} + }), + "refused": worded(func(o conditions.Observation) words { + return words{Headline: "The bus refuses some messages", + Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.", + Resolved: "Resolved: the bus takes the messages again"} + }), + "stream-wrong": worded(func(o conditions.Observation) words { + return words{Headline: "Part of the bus's storage is wrong", + Needs: "check the machine the bus runs on; the details say what is missing.", + Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.", + Resolved: "The bus's storage is right again"} + }), + "archives-unheld": worded(func(o conditions.Observation) words { + return words{Headline: "Kept builds are about to be cleaned up", + Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.", + Resolved: "Resolved: every kept build is claimed"} + }), + "archives-missing": worded(func(o conditions.Observation) words { + return words{Headline: "Kept builds are missing", + Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.", + Resolved: "Resolved: the kept builds are back"} + }), +} + +// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days". +func humanDuration(d time.Duration) string { + switch { + case d < 2*time.Minute: + return "a minute" + case d < 2*time.Hour: + return fmt.Sprintf("%d minutes", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%d hours", int(d.Hours())) + } + return fmt.Sprintf("%d days", int(d.Hours()/24)) +} + +// repoName is a repository as a person says it: its name without its owner. +func repoName(repository string) string { + if i := strings.LastIndex(repository, "/"); i >= 0 { + return repository[i+1:] + } + return repository +} + +// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more". +func namesWords(names []string, most int) string { + switch { + case len(names) == 0: + return "" + case len(names) == 1: + return names[0] + case len(names) <= most: + return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] + } + return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most) +} + +// planModules are the modules a walk moves, sorted. +func planModules(p inventory.Plan) []string { + out := make([]string, 0, len(p.Modules)) + for m := range p.Modules { + out = append(out, m) + } + sort.Strings(out) + return out +} + +// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else +// their count and the repository. +func deliveryWhat(modules []string, repository string) string { + if len(modules) > 0 && len(modules) <= 2 { + return namesWords(modules, 2) + } + if len(modules) > 2 { + return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository)) + } + return repoName(repository) +} + +// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery". +func deliveryName(modules []string, repository string) string { + if len(modules) > 0 && len(modules) <= 2 { + return namesWords(modules, 2) + " delivery" + } + return repoName(repository) + " delivery" +} + +// walkWaitingWords explains a walk waiting for its delivery's word. +func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string { + what := deliveryWhat(w.modules, w.repository) + if severity == conditions.Urgent { + return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+ + "delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in)) + } + return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+ + "decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.", + what, humanDuration(in), humanDuration(waitUrgentAfter)) +} + +// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. +func waitingNeeds(severity conditions.Severity) string { + if severity == conditions.Urgent { + return "start it, or stop it." + } + return "" +} + +// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since +// the module that should have said go is the one not answering. +func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action { + if severity != conditions.Urgent { + return nil + } + return []conditions.Action{ + {Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}}, + {Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}}, + } +} + +// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its +// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it. +func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) { + var actions []conditions.Action + for _, r := range rs { + if strings.Contains(r.Reason, "relogin needed") { + return fmt.Sprintf("log out of every session on %s and log in again.", node), nil + } + if r.Kind == link.KindUnit && len(actions) < 2 { + scope := "system" + if strings.Contains(r.Reason, "account's own") { + scope = "user" + } + label := "Restart" + if len(actions) > 0 { + label = "Restart " + unitPlainWords(r.Target) + } + actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart", + Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}}) + } + } + if len(actions) > 0 { + return "restart it; if it fails again, the details say why.", actions + } + return "", nil +} + +// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it. +func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string, + actions []conditions.Action) { + repository, _, _ := strings.Cut(l.ID, "@") + name := repoName(repository) + held := l.State + if held == "" { + held = "held" + } + long := "too long" + if d, err := time.ParseDuration(l.For); err == nil { + long = "for " + humanDuration(d) + } + if o.Resolver == conditions.ResolverOperator { + switch held { + case "held": + needs = "release it, or stop it." + actions = []conditions.Action{ + {Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}}, + {Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}, + } + case "ready", "checked": + needs = "merge its pull request, or close it." + default: + needs = "stop it, or read the details to see what it waits for." + actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}} + } + } + return fmt.Sprintf("Delivery of %s %s %s", name, held, long), + fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long), + fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions +} + +// causeWords is a hand-act's cause as a person says it. +func causeWords(cause string) string { + return strings.NewReplacer(".", " ", "_", " ").Replace(cause) +} + +// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)". +func unitPlainWords(unit string) string { + for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer", + ".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} { + if name, ok := strings.CutSuffix(unit, suffix); ok { + if what == "" { + return strings.ReplaceAll(name, ".", " ") + } + return strings.ReplaceAll(name, ".", " ") + " (" + what + ")" + } + } + return strings.ReplaceAll(unit, ".", " ") +} + +// resourcePlainWords is what is wrong with one of a module's resources, as a person says it. +func resourcePlainWords(r inventory.ResourceHealth) string { + if r.Kind == link.KindUnit { + return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason) + } + what := "its " + strings.ReplaceAll(r.Kind, "-", " ") + switch r.Reason { + case "restarting": + return what + " keeps restarting" + case "down": + return what + " is not running" + } + if r.Check != "" { + return what + " fails its " + r.Check + " check" + } + return what + " is not healthy" +} + +// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be +// systemd's result, such as exit-code, or a sentence carrying it). +func unitReasonWords(reason string) string { + switch { + case strings.Contains(reason, "start-limit"): + return "failed too often to be started again" + case strings.Contains(reason, "exit-code"): + return "stopped with an error" + case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"): + return "crashed" + case strings.Contains(reason, "timeout"): + return "timed out" + } + return "failed" +} diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go new file mode 100644 index 00000000..8a2c653a --- /dev/null +++ b/cmd/mesh-controller/plain_words_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The conditions the operator read on 2026-10-08, raised again from the same facts (novox/hq ADR 0253): +// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a +// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them. + +// plainExample checks one finding's words as the keeper will keep them: plain, within their bounds, +// opened by the verdict, and saying what the operator needs; and its actions by label. +func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) { + t.Helper() + machines := append([]string{o.Machine}, o.Also...) + w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, + Actions: o.Actions} + if why, ok := conditions.PlainWords(w, machines...); !ok { + t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w) + } + said := conditions.Verdict(o.Needs, o.Explanation) + if o.Headline != headline || said != explanation { + t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation) + } + var labels []string + for _, a := range o.Actions { + labels = append(labels, a.Label) + } + if strings.Join(labels, ",") != strings.Join(actions, ",") { + t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions) + } + quiet := "" + if o.Needs == "" && o.Severity == conditions.Warning { + quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)" + } + t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet, + said, labels, o.Resolved) +} + +// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for +// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why; +// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no +// popup; past them, the operator starts or stops it from the notification. +func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) { + now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC) + f := calm(now) + f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee", + awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}} + got := watchWaits(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + plainExample(t, got[0], "openrazer delivery waiting to start", + "Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+ + "mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+ + "for you if it still waits after 4 hours.") + if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") { + t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary) + } + + // Past four hours it is urgent, and offers the controller's own answers. + f.waits[0].since = now.Add(-5 * time.Hour) + got = watchWaits(f) + plainExample(t, got[0], "openrazer delivery waiting to start", + "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+ + "module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+ + "be stuck.", "Start", "Stop") + if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" { + t.Errorf("start: %+v", a) + } + if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" { + t.Errorf("stop: %+v", a) + } + + // Many modules are counted, not listed in the headline. + f.waits[0].modules = []string{"a", "b", "c", "d"} + got = watchWaits(f) + if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") { + t.Errorf("%+v", got[0]) + } +} + +// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the +// account's own service manager (exit-code)". The operator restarts it from the notification. +func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) { + o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit, + Resource: "openrazer-daemon", Target: "openrazer-daemon.service", + Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}}) + plainExample(t, o, "openrazer not working on g14", + "Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+ + "service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart") + if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" || + a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" { + t.Errorf("restart: %+v", a) + } + // An account waiting for a new login (ADR 0252) asks for the login, which no button can give. + o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account", + Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}}) + if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 { + t.Errorf("relogin: %q %+v", o.Needs, o.Actions) + } +} + +// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places — +// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …" +func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) { + o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{ + {Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"}, + {Unit: "greenclip.service", Scope: "user", Result: "exit-code"}}}) + if !raise { + t.Fatal("not raised") + } + plainExample(t, o, "3 failed services on shanks", + "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+ + "mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+ + "mesh does not repair them.", "Silence for a week") +} + +// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through +// the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup. +func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) { + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + f := calm(now) + for i := 0; i < 35; i++ { + a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), "push") + a.Verb, a.By = "push", "g14.node-tools, through the mesh-controller seat" + f.handActs = append(f.handActs, a) + } + got := watchHandActs(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + plainExample(t, got[0], "Push keeps being fixed by hand", + "Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+ + "wanted for it. Nothing is broken now.") +} + +// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been +// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …". +func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) { + got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", + Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}}) + plainExample(t, got[0], "Delivery of hq held for 36 hours", + "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.", + "Release", "Stop") + if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" { + t.Errorf("release: %+v", a) + } +} + +// **Every kind the controller raises has plain words**, and its words are plain for a subject of every +// shape it is raised with. +func TestEveryWordingIsPlain(t *testing.T) { + subjects := []conditions.Observation{ + {Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"}, + {Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"}, + {Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}, + {Scope: conditions.ScopeProvider, ID: "postgres.anchor.app_db", Machine: "anchor"}, + {Scope: conditions.ScopePlan, ID: "plan-1791454185265004861"}, + {Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"}, + {Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"}, + {Scope: conditions.ScopeBus, ID: "mesh"}, + {Scope: conditions.ScopeSeat, ID: "node-backup.ace", Machine: "ace"}, + {Scope: conditions.ScopeMesh, ID: "hand-acts.disk-load"}, + {Scope: conditions.ScopeBuild, ID: "openrazer.g14", Machine: "g14"}, + } + for kind, fn := range plainWordings { + if !conditions.Worded(kind) { + t.Errorf("%s is not registered", kind) + } + for _, s := range subjects { + s.Kind = kind + w := fn(s) + w.Explanation = conditions.Verdict(w.Needs, w.Explanation) + if why, ok := conditions.PlainWords(w, s.Machine); !ok { + t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w) + } + } + } +} diff --git a/cmd/mesh-controller/sayable_test.go b/cmd/mesh-controller/sayable_test.go index d7a5fc67..6fb86b7c 100644 --- a/cmd/mesh-controller/sayable_test.go +++ b/cmd/mesh-controller/sayable_test.go @@ -29,7 +29,20 @@ func TestMain(m *testing.M) { conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { unsaid.note(o, field, r) } + conditions.Unworded = func(o conditions.Observation, why string) { + if o.Source != "test" { // a kind a test makes up to exercise the keeper + unworded.add(o, why) + } + } code := m.Run() + if said := unworded.all(); len(said) > 0 { + fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+ + "words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+ + "reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n ")) + if code == 0 { + code = 1 + } + } if said := unsaid.all(); len(said) > 0 { fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+ "(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n", @@ -74,6 +87,34 @@ func (u *unsayable) all() []string { return out } +// unworded collects, across the suite, every finding said in borrowed words (ADR 0253). +var unworded wordless + +type wordless struct { + mu sync.Mutex + seen map[string]bool +} + +func (u *wordless) add(o conditions.Observation, why string) { + u.mu.Lock() + defer u.mu.Unlock() + if u.seen == nil { + u.seen = map[string]bool{} + } + u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true +} + +func (u *wordless) all() []string { + u.mu.Lock() + defer u.mu.Unlock() + var out []string + for s := range u.seen { + out = append(out, s) + } + sort.Strings(out) + return out +} + // linted is a producer's findings, held to the content rule as a keeper would hold them: for a test // that reads a producer's findings without raising them. func linted(obs []conditions.Observation) []conditions.Observation { @@ -85,6 +126,18 @@ func linted(obs []conditions.Observation) []conditions.Observation { if r, ok := outward.Check(o.Summary, machines...); !ok { unsaid.note(o, "summary", r) } + switch { + case o.Headline != "": + w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved} + if w.Resolved == "" { + w.Resolved = "Resolved" + } + if why, ok := conditions.PlainWords(w, machines...); !ok { + unworded.add(o, "its source's words are not plain: "+why) + } + case o.Kind != "" && !conditions.Worded(o.Kind): + unworded.add(o, "the kind "+o.Kind+" has no plain words") + } } return obs } diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 1ac39ca4..1ea068d3 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -322,6 +322,10 @@ func watchPlans(f *signalFacts) []conditions.Observation { continue } out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled", + Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway", + Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+ + "usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))), + Resolved: "Delivery of " + repoName(p.repository) + " moves again", Severity: conditions.Warning, Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s", p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"), @@ -353,7 +357,12 @@ func watchWaits(f *signalFacts) []conditions.Observation { Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+ "delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository, short(w.commit), ago(in), w.awaits, short(w.commit), w.id), - Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits)}) + Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits), + Headline: deliveryName(w.modules, w.repository) + " waiting to start", + Explanation: walkWaitingWords(w, in, severity), + Needs: waitingNeeds(severity), + Actions: waitingActions(w, severity), + Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"}) } return out } @@ -681,7 +690,12 @@ func watchHandActs(f *signalFacts) []conditions.Observation { Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning, Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted), - Said: strings.Join(acts, "; ")}) + Said: strings.Join(acts, "; "), + Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand", + Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is "+ + "wanted for it. Nothing is broken now.", causeWords(cause), repeated[cause], + int(handActsWithin.Hours()/24)), + Resolved: "Resolved: no more hand repairs of " + causeWords(cause)}) } return out } diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 72233297..6f73c860 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -150,6 +150,8 @@ type planFacts struct { type waitFacts struct { id, repository, commit, awaits string since time.Time + // modules are the modules the walk moves, as a person names the delivery. + modules []string } type loopFacts struct { @@ -451,7 +453,7 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ( // S16's, whatever mesh-delivery says or does not say. if p.Waiting() { waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit, - awaits: p.Delivery.Awaits, since: p.Created}) + awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p)}) continue } _, paused := pausedWaiting(p, pause, now) diff --git a/internal/catalogue/foundation_manifests_test.go b/internal/catalogue/foundation_manifests_test.go index 49993e6a..a78fe472 100644 --- a/internal/catalogue/foundation_manifests_test.go +++ b/internal/catalogue/foundation_manifests_test.go @@ -177,11 +177,19 @@ func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) { // wrong on every node whose assignment differs, and wrong for a second reason on a node given the // port (ADR 0100). Composed through the whole path, because what proves the placeholder resolves // in an `env` at all is a declaration, not a manifest. +// forgeBuilt is every artifact the forge's manifest builds: its code, and since hq ADR 0251 the npm +// registry it serves beside it. A manifest resolves only against all of what it asked to be built. +func forgeBuilt() []Built { + var out []Built + for _, name := range []string{"code", "npm-registry"} { + out = append(out, Built{Name: name, Kind: ArtifactBundle, + Reference: ArtifactStoreScheme + "gitea/" + name + "/blobs/" + bundleDigest, Digest: bundleDigest}) + } + return out +} + func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) { - forge, err := catalogueManifest(t, "gitea").Resolve([]Built{{ - Name: "code", Kind: ArtifactBundle, - Reference: ArtifactStoreScheme + "gitea/code/blobs/" + bundleDigest, Digest: bundleDigest, - }}) + forge, err := catalogueManifest(t, "gitea").Resolve(forgeBuilt()) if err != nil { t.Fatalf("the forge's manifest does not resolve against its own build: %v", err) } @@ -234,10 +242,7 @@ func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) { func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any { t.Helper() forge := catalogueManifest(t, "gitea") - resolved, err := forge.Resolve([]Built{{ - Name: "code", Kind: ArtifactBundle, - Reference: ArtifactStoreScheme + "gitea/code/blobs/" + bundleDigest, Digest: bundleDigest, - }}) + resolved, err := forge.Resolve(forgeBuilt()) if err != nil { t.Fatalf("the forge's manifest does not resolve against its own build: %v", err) } diff --git a/internal/conditions/condition.go b/internal/conditions/condition.go index 500e7937..cc253a6a 100644 --- a/internal/conditions/condition.go +++ b/internal/conditions/condition.go @@ -125,8 +125,19 @@ type Condition struct { Kind string `json:"kind"` Subject Subject `json:"subject"` Severity Severity `json:"severity"` - // Summary is one line in the mesh's words. + // Summary is one line in the mesh's words, for whoever looks closer: it may name plans, commits and + // the verbs that act. Summary string `json:"summary"` + // Headline, Explanation and Resolved are what the operator reads, in plain words (novox/hq ADR 0253, + // plain.go): a few words naming the thing and what is wrong; one or two sentences on what happened, + // what it means and whether to act; the one line said when it clears. + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Resolved string `json:"resolved"` + // Needs is what the operator does about it, one sentence; empty when nothing (the explanation then + // opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go). + Needs string `json:"needs"` + Actions []Action `json:"actions"` // Evidence is the newest observations, at most KeptEvidence, newest first. Evidence []Evidence `json:"evidence"` // Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. @@ -175,6 +186,13 @@ type Observation struct { Also []string Severity Severity Summary string + // Headline, Explanation and Resolved are the plain words the operator reads (plain.go). Left empty, + // the wording registered for Kind says them. + Headline string + Explanation string + Resolved string + Needs string + Actions []Action // Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes // here, never in Summary**: an address, a socket's error, a path or a name with its domain is // kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the diff --git a/internal/conditions/plain.go b/internal/conditions/plain.go new file mode 100644 index 00000000..19ce4ee0 --- /dev/null +++ b/internal/conditions/plain.go @@ -0,0 +1,341 @@ +package conditions + +// Plain words (novox/hq ADR 0253): **a condition carries what the operator reads, in plain words, beside +// what an agent reads.** The summary is one line in the mesh's words for whoever looks closer — it names +// plans, commits and the verbs that act, and stays so. The operator reads a notification between other +// work: a popup that said "the walk of novox/mesh-catalog a6385479 has waited 56m0s for mesh-delivery's +// word to start: `mesh-delivery.show` …" was a wall of identifiers nobody could act on. So every condition +// also carries: +// +// - a **headline**: a few plain words naming the thing and what is wrong ("openrazer not running on +// g14"), the title of every message about it; +// - an **explanation**: one or two plain sentences — what happened, what it means for the operator, +// and whether they need to do anything; +// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14"); +// - a **verdict**, which every explanation opens with: "Nothing for you to do." or "Needs you:" and one +// concrete thing the operator can do themselves (needs). Never "have an agent …": the operator is not +// asked to open a session to understand or answer a notification; +// - **actions**: what the operator may answer from the notification itself (Release, Stop, Restart, +// Silence), each the seat verb the operator channel calls when it is chosen, naming the operator. +// +// They are made here, where the condition is made, so every channel gets them — the desktop today, +// others later — and none has to guess what a key means. A producer may say them itself; otherwise the +// wording registered for its kind says them; otherwise a plain sentence made from its scope does, and a +// test suite is told (Unworded) so the kind gets words of its own. Each is held to the plain rule (Plain): +// no identifiers, hashes, keys, verb syntax, markup or clock times — a message names no time of its own, +// since the channel says when, in the operator's time. + +import ( + "fmt" + "regexp" + "strings" + "sync" + + "github.com/novox/mesh-controller/internal/outward" +) + +// Words are what the operator reads of a condition. Explanation is said after the verdict, which +// plainly writes from Needs: empty is "Nothing for you to do.", else "Needs you: " and Needs. +type Words struct { + Headline string + Explanation string + Resolved string + Needs string + Actions []Action +} + +// Action is one answer the operator may give from a notification: a label, and the seat verb the +// operator channel calls with these arguments (and a why naming the operator and the label) when it is +// chosen. Machine is set for a seat every machine holds. An argument "why" given empty is the operator +// channel's to fill: it names the operator, the channel and the label chosen. +type Action struct { + Label string `json:"label"` + Verb string `json:"verb"` + Machine string `json:"machine,omitempty"` + Arguments map[string]string `json:"arguments,omitempty"` +} + +// The two verdicts an explanation opens with. +const ( + NothingToDo = "Nothing for you to do." + NeedsYou = "Needs you:" +) + +// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the +// answer to a condition the operator decided to live with. +func SilenceAction(key string) Action { + return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions", + Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}} +} + +// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences. +const ( + HeadlineMax = 60 + ExplanationMax = 420 + NeedsMax = 120 +) + +var ( + wordingsMu sync.RWMutex + wordings = map[string]func(Observation) Words{} +) + +// Wording registers the plain words of one condition kind. +func Wording(kind string, words func(Observation) Words) { + wordingsMu.Lock() + defer wordingsMu.Unlock() + wordings[kind] = words +} + +// Worded says whether a kind has words of its own. +func Worded(kind string) bool { + wordingsMu.RLock() + defer wordingsMu.RUnlock() + _, ok := wordings[kind] + return ok +} + +// Unworded is told of every observation said in borrowed words: its kind has none registered, or the +// words it was given break the plain rule. The keeper says it plainly anyway; a test suite sets this to +// fail the producer. +var Unworded func(o Observation, why string) + +// The plain rule's shapes. +var ( + hexID = regexp.MustCompile(`\b[0-9a-f]{7,40}\b`) + numberedID = regexp.MustCompile(`\b[a-z]+-[0-9]{6,}\b`) + dotted = regexp.MustCompile(`\b[a-z][a-z0-9_-]*\.[a-z][a-z0-9_-]*\b`) + goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`) + clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`) + flag = regexp.MustCompile(`(^|\s)--?[a-z]`) + agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`) + markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)") +) + +// Plain says whether text is plain words the operator reads at a glance, and if not, what is not: an +// identifier (a hash, a numbered id, a dotted name such as a key or a verb), a Go duration, a clock time +// or date (the channel says when, in the operator's time), a command's flag, markup, a line break, or +// anything the operator's channel withholds (ADR 0234 §6). machines are names that may appear. +func Plain(text string, machines ...string) (string, bool) { + if strings.TrimSpace(text) == "" { + return "nothing is said", false + } + if strings.ContainsAny(text, "\n\r\t") { + return "a line break", false + } + if m := markup.FindString(text); m != "" { + return "markup (" + m + ")", false + } + for _, shape := range []struct { + re *regexp.Regexp + what string + }{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"}, + {clockTime, "a clock time or date"}, {flag, "a command's flag"}} { + if m := shape.re.FindString(text); m != "" { + if shape.re == hexID && !strings.ContainsAny(m, "0123456789") { + continue // a word of letters a to f only + } + return shape.what + " (" + strings.TrimSpace(m) + ")", false + } + } + if r, ok := outward.Check(text, machines...); !ok { + return r.String(), false + } + return "", true +} + +// PlainWords checks all three, with their bounds. +func PlainWords(w Words, machines ...string) (string, bool) { + if why, ok := Plain(w.Headline, machines...); !ok { + return "headline: " + why, false + } + if len(w.Headline) > HeadlineMax { + return fmt.Sprintf("headline: longer than %d characters", HeadlineMax), false + } + if why, ok := Plain(w.Explanation, machines...); !ok { + return "explanation: " + why, false + } + if len(w.Explanation) > ExplanationMax { + return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false + } + for _, m := range agentWord.FindAllString(w.Explanation, -1) { + if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not + return "explanation: sends the operator elsewhere (" + m + ")", false + } + } + if w.Needs != "" { + if why, ok := Plain(w.Needs, machines...); !ok { + return "needs: " + why, false + } + if m := agentWord.FindString(w.Needs); m != "" { + return "needs: not something the operator does themselves (" + m + ")", false + } + if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") { + return fmt.Sprintf("needs: one sentence of at most %d characters, ending in a full stop", NeedsMax), false + } + } + for _, a := range w.Actions { + if a.Label == "" || len(a.Label) > 24 || a.Verb == "" { + return fmt.Sprintf("action %q: a short label and a verb", a.Label), false + } + } + if why, ok := Plain(w.Resolved, machines...); !ok { + return "resolved: " + why, false + } + if len(w.Resolved) > HeadlineMax+20 { + return fmt.Sprintf("resolved: longer than %d characters", HeadlineMax+20), false + } + return "", true +} + +// plainly gives an observation its plain words: its own, its kind's, or its scope's. +func plainly(o Observation) Observation { + machines := append([]string{o.Machine}, o.Also...) + given := Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, + Actions: o.Actions} + var w Words + from := "" + switch { + case given.Headline != "": + w, from = given, "its source" + default: + wordingsMu.RLock() + fn := wordings[o.Kind] + wordingsMu.RUnlock() + if fn != nil { + w, from = fn(o), "the wording of "+o.Kind + if given.Explanation != "" { + w.Explanation = given.Explanation + } + if given.Resolved != "" { + w.Resolved = given.Resolved + } + if given.Needs != "" { + w.Needs = given.Needs + } + if len(given.Actions) > 0 { + w.Actions = given.Actions + } + } + } + if w.Headline != "" && w.Resolved == "" { + w.Resolved = "Resolved: " + lowerFirst(w.Headline) + } + if from == "" { + if Unworded != nil { + Unworded(o, "the kind "+o.Kind+" has no plain words") + } + w = scopeWords(o) + } else if why, ok := PlainWords(w, machines...); !ok { + if Unworded != nil { + Unworded(o, from+" is not plain: "+why) + } + w = scopeWords(o) + } + o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions + o.Explanation = Verdict(w.Needs, w.Explanation) + return o +} + +// Verdict is an explanation opened by its verdict. +func Verdict(needs, explanation string) string { + if needs == "" { + return strings.TrimSpace(NothingToDo + " " + explanation) + } + return strings.TrimSpace(NeedsYou + " " + needs + " " + explanation) +} + +// Escalated is what a condition needs once a healer gave up on it, where its words needed nothing. +const Escalated = "the mesh tried to repair this and could not; read the details to decide what to do." + +// escalatedWords turns "Nothing for you to do." into "Needs you:" once a healer gave up. +func escalatedWords(c *Condition) { + if c.Escalated() && c.Needs == "" { + c.Needs = Escalated + c.Explanation = Verdict(c.Needs, strings.TrimSpace(strings.TrimPrefix(c.Explanation, NothingToDo))) + } +} + +// scopeWords is what is said of a kind with no words of its own: what kind of thing, and which one +// where its name is a name (a machine, a module), never an id. +func scopeWords(o Observation) Words { + thing := ThingWords(o) + what := strings.ReplaceAll(o.Kind, "-", " ") + w := Words{ + Headline: Capital(thing) + " needs a look", + Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s.", what, thing), + Resolved: "Resolved: " + thing + " is fine again", + Needs: ResolverNeeds(o), + } + if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok { + w = Words{Headline: "Something in the mesh needs a look", + Explanation: "The mesh noticed a problem it has no plain words for yet.", + Resolved: "Resolved: the mesh is fine again", Needs: ResolverNeeds(o)} + } + return w +} + +// ThingWords is what a condition is about, in plain words: "the machine ace", "openrazer on g14", "a +// delivery". An id that is not a name (a plan's, a delivery's, a call's) is never said. +func ThingWords(o Observation) string { + switch o.Scope { + case ScopeMachine: + if o.Machine != "" { + return o.Machine + } + return "a machine" + case ScopeModule: + module := strings.TrimSuffix(o.ID, "."+o.Machine) + if o.Machine != "" && module != "" && !strings.Contains(module, ".") { + return module + " on " + o.Machine + } + return "a module" + case ScopeProvider: + if module, _, _ := strings.Cut(o.ID, "."); module != "" && o.Machine != "" { + return module + " on " + o.Machine + } + return "a provider" + case ScopePlan: + return "a walk" + case ScopeDelivery: + return "a delivery" + case ScopeBuild: + return "a build" + case ScopeCall: + return "a tool call" + case ScopeMerge: + return "a merge" + case ScopeSeat: + return "a seat's holder" + case ScopeBus: + return "the bus" + case ScopeCore: + return "the controller" + case ScopeProbe: + return "the self-check" + } + return "the mesh" +} + +// ResolverNeeds is what the operator needs to do, from who resolves it: nothing, unless only a person can. +func ResolverNeeds(o Observation) string { + if o.Resolver == ResolverOperator { + return "the mesh does not repair this by itself; read the details to decide what to do." + } + return "" +} + +// Capital is s with its first letter upper case. +func Capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func lowerFirst(s string) string { + if s == "" || (len(s) > 1 && strings.ToUpper(s[:2]) == s[:2]) { + return s + } + return strings.ToLower(s[:1]) + s[1:] +} diff --git a/internal/conditions/plain_test.go b/internal/conditions/plain_test.go new file mode 100644 index 00000000..9a129c67 --- /dev/null +++ b/internal/conditions/plain_test.go @@ -0,0 +1,163 @@ +package conditions + +import ( + "encoding/json" + "strings" + "testing" +) + +// **The plain rule** (novox/hq ADR 0253): what the operator reads names no identifier, no code and no time +// of its own; the words conditions are made of pass. +func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) { + refused := map[string]string{ + "the walk of novox/mesh-catalog a6385479 waits": "a hash", + "plan plan-1791454185265004861 is waiting": "a hash", + "see `mesh-delivery.show` for why": "markup", + "call mesh-delivery.show for why": "a dotted name", + "key plan.x.waiting": "a dotted name", + "waited 56m0s": "a duration in code", + "since 10:40": "a clock time", + "since 2026-10-08": "a clock time", + "raised at noon UTC": "a clock time", + "start it with plans go --why": "a command's flag", + "one line\nand another": "a line break", + "the resolver at 10.77.0.1 is silent": "", + "": "nothing", + "**bold**": "markup", + "storage-media.mount failed": "a dotted name", + "the store at /var/lib/mesh is full": "", + "a commit 0123abc landed": "a hash", + } + for text, want := range refused { + why, ok := Plain(text, "g14") + if ok { + t.Errorf("%q passed", text) + continue + } + if want != "" && !strings.HasPrefix(why, want) { + t.Errorf("%q: refused for %q, want %q", text, why, want) + } + } + for _, text := range []string{ + "openrazer delivery waiting to start", + "The change to openrazer is merged and built, and has waited 56 minutes for mesh-delivery to let it start.", + "3 failed services on shanks", + "On shanks, mnt-recalbox (a mount), storage-media (a mount) and greenclip failed.", + "Name lookups fail on g14", + "Retired data of postgres on anchor waits for app_db", + "Push keeps being fixed by hand", + } { + if why, ok := Plain(text, "g14", "shanks", "anchor"); !ok { + t.Errorf("%q refused: %s", text, why) + } + } +} + +// **A condition carries its plain words**: a producer's own; else its kind's; else its scope's — and the +// event says them beside the summary, for every channel. +func TestAConditionCarriesItsPlainWords(t *testing.T) { + k, _, told, _ := keeper(t) + before := Unworded + var unworded []string + Unworded = func(o Observation, why string) { unworded = append(unworded, why) } + t.Cleanup(func() { Unworded = before }) + + Wording("test-silent", func(o Observation) Words { + return Words{Headline: o.Machine + " is not answering", Explanation: "The mesh has not heard from it.", + Resolved: o.Machine + " answers again"} + }) + c, err := k.Observe(t.Context(), Observation{Scope: ScopeMachine, ID: "ace", Kind: "test-silent", Machine: "ace", + Severity: Warning, Source: "S1", Summary: "ace has not been heard from since 12:00 UTC (bound 3m0s)"}) + if err != nil { + t.Fatal(err) + } + if c.Headline != "ace is not answering" || c.Resolved != "ace answers again" || c.Explanation == "" { + t.Fatalf("the kind's words were not kept: %+v", c) + } + if c.Summary != "ace has not been heard from since 12:00 UTC (bound 3m0s)" { + t.Errorf("the summary was changed: %q", c.Summary) + } + said := settled(t, told, 1) + body, _ := json.Marshal(said[0]) + for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"Nothing for you to do. The mesh has not heard from it."`, + `"resolved":"ace answers again"`} { + if !strings.Contains(string(body), field) { + t.Errorf("the event does not carry %s: %s", field, body) + } + } + + // A producer's own words win; a resolved line it leaves out is made from its headline. + c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Kind: "test-silent", + Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy", + Headline: "openrazer not working on g14", Explanation: "Its service stopped with an error."}) + if c.Headline != "openrazer not working on g14" || c.Resolved != "Resolved: openrazer not working on g14" { + t.Errorf("the producer's words: %+v", c) + } + if len(unworded) != 0 { + t.Errorf("words that were plain were reported: %v", unworded) + } + + // Words that are not plain, and a kind with none, are said from the scope — and reported. + c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded", + Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"}) + if c.Headline != "Openrazer on g14 needs a look" || !strings.HasPrefix(c.Explanation, NothingToDo) { + t.Errorf("the scope's words: %+v", c) + } + c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent", + Severity: Warning, Source: "S16", Summary: "the walk waits", Headline: "plan-1791454185265004861 waits"}) + if strings.Contains(c.Headline, "plan-") || c.Headline != "A walk needs a look" { + t.Errorf("an id reached the headline: %+v", c) + } + if len(unworded) != 2 || !strings.Contains(unworded[0], "no plain words") || !strings.Contains(unworded[1], "not plain") { + t.Errorf("what was said in borrowed words was not reported: %v", unworded) + } +} + +// **An explanation opens with its verdict, and never sends the operator to an agent.** +func TestEveryExplanationOpensWithItsVerdict(t *testing.T) { + if got := Verdict("", "It may be asleep."); got != "Nothing for you to do. It may be asleep." { + t.Errorf("%q", got) + } + if got := Verdict("release it, or stop it.", "It is held."); got != "Needs you: release it, or stop it. It is held." { + t.Errorf("%q", got) + } + for _, w := range []Words{ + {Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"}, + {Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"}, + {Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"}, + {Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"}, + {Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}}, + } { + if _, ok := PlainWords(w); ok { + t.Errorf("passed: %+v", w) + } + } + if why, ok := PlainWords(Words{Headline: "push keeps being fixed by hand", Explanation: "A person repaired " + + "push by hand 35 times.", Resolved: "Resolved", Needs: "release it, or stop it.", + Actions: []Action{{Label: "Release", Verb: "mesh-delivery.release"}}}); !ok { + t.Errorf("refused: %s", why) + } +} + +// **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the +// budget is spent, the operator is needed. +func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) { + k, _, _, _ := keeper(t) + ctx := t.Context() + o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1", + Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."} + if _, err := k.Observe(ctx, o); err != nil { + t.Fatal(err) + } + c, _, err := k.Escalate(ctx, o.Key(), Attempt{What: "budget spent", Outcome: "escalated", By: "healer H1"}) + if err != nil { + t.Fatal(err) + } + if want := "Needs you: " + Escalated + " It may be asleep."; c.Explanation != want || c.Needs != Escalated { + t.Fatalf("escalated: %q, want %q", c.Explanation, want) + } + c, _ = k.Observe(ctx, o) + if strings.Count(c.Explanation, Escalated) != 1 || strings.Contains(c.Explanation, NothingToDo) { + t.Errorf("seen again after escalation: %q", c.Explanation) + } +} diff --git a/internal/conditions/store.go b/internal/conditions/store.go index 671a9de0..ebecde5b 100644 --- a/internal/conditions/store.go +++ b/internal/conditions/store.go @@ -175,6 +175,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) return Condition{}, err } o = k.sayable(o) + o = plainly(o) key := o.Key() for i := 0; i < tries; i++ { now := k.now().UTC() @@ -188,7 +189,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) } if !found { c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also}, - Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}}, + Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation, + Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}}, Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1, Resolver: orSelf(o.Resolver)} change := ChangeRaised @@ -238,6 +240,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) // The kind as the source says it now: a source that gave the same key a kind of its own since // (a probe's finding split out for a healer) is read by that kind from its next observation. c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now + c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions + escalatedWords(&c) if o.Machine != "" { c.Subject.Machine = o.Machine } @@ -335,6 +339,7 @@ func (k *Keeper) Escalate(ctx context.Context, key string, a Attempt) (Condition changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome}) c.Resolver = ResolverOperator } + escalatedWords(c) return changes }) }