package main // The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator // reads — a headline, one or two sentences on what it means and whether to act, and the line said when it // clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer. // // A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's // cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test // suite is held to having words, and to their being plain (plain_words_test.go). import ( "fmt" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) type words = conditions.Words // worded is a wording that needs only the condition's subject. func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn } func init() { for kind, fn := range plainWordings { conditions.Wording(kind, fn) } } // machineOr is the machine a condition concerns, or what to say when none is named. func machineOr(o conditions.Observation, none string) string { if o.Machine != "" { return o.Machine } return none } // idPart is the n-th dotted part of a condition's id, or "". func idPart(o conditions.Observation, n int) string { parts := strings.Split(o.ID, ".") if n < len(parts) { return parts[n] } return "" } // dataWords is " of " for a data condition keyed by machine, module and item. func dataWords(o conditions.Observation) (what, module string) { module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ") switch { case module != "" && item != "": return item + " of " + module, module case module != "": return "data of " + module, module } return "data", "a module" } var plainWordings = map[string]func(conditions.Observation) words{ // Machines and their node-engine. "silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") w := words{Headline: m + " is not answering", Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+ "normal.", m), Resolved: m + " answers again"} if o.Severity == conditions.Urgent { w.Needs = fmt.Sprintf("check that %s is on and online.", m) w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+ "nothing changes anywhere until it is back.", m) } return w }), "sent-not-reported": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + " has not confirmed a change", Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+ "It usually catches up by itself.", m, m), Resolved: m + " applied the change"} }), "tools-silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Tools on " + m + " are not answering", Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+ "the mesh until it does. A machine asleep is normal.", m, m), Resolved: "Tools on " + m + " answer again"} }), kindAwaitingPush: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + " waits for new instructions", Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.", conditions.Capital(m)), Resolved: m + " got its new instructions"} }), "declaration-refused": worded(func(o conditions.Observation) words { if o.Scope == conditions.ScopeMesh { return words{Headline: "No machine can get new instructions", Needs: "decide whether to undo the last catalogue change; the details say what fails.", Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " + "made. Machines keep running what they have.", Resolved: "Machines can get new instructions again"} } m := machineOr(o, "a machine") return words{Headline: m + " cannot get new instructions", Needs: "decide whether to undo the last catalogue change; the details say what fails.", Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+ "reaches it. It keeps running what it has.", m), Resolved: m + " can get new instructions again"} }), "own-address-banned": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + " has banned the mesh", Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m), Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+ "mesh is locked out of it.", m), Resolved: m + " no longer bans the mesh"} }), "core-behind": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + " runs an old node-engine", Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+ "works, and gets the newer one with a later delivery.", conditions.Capital(m)), Resolved: m + " runs the current node-engine"} }), kindMachineUnits: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Failed services on " + m, Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m), Explanation: "No module manages them, so the mesh does not repair them.", Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}} }), kindMachineNetwork: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + " has a network problem", Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+ "route. It often passes again once a network change settles.", conditions.Capital(m)), Resolved: m + "'s network is fine again"} }), kindNetworkRewritten: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + "'s network settings were changed", Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+ "client. Mesh names may not resolve there until it is undone.", m), Resolved: m + "'s network settings are the mesh's again"} }), kindNetworkUnreachable: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: m + " cannot be reached", Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+ "asleep is normal.", m), Resolved: m + " can be reached again"} }), // A pending assignment that was not made (novox/hq issue 325, ADR 0261). kindPendingEnded: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") module := idPart(o, 1) if module == "" { module = "a module" } return words{Headline: module + " was not put on " + m, Needs: "decide whether to build it again or take the assignment back; the details say why.", Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+ "register it, so it was not made.", module, m), Resolved: "Resolved: " + module + " on " + m + " is settled"} }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, Needs: "decide where its data lives; the details name both places.", Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+ "kept the old one because switching would leave the data behind.", m), Resolved: "Resolved: the data source on " + m + " is decided"} }), kindBindingMoving: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source moves on " + m, Needs: "move its data to the new provider before the next push.", Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m), Resolved: "Resolved: the data source move on " + m + " is settled"} }), kindBindingMoved: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source moved on " + m, Needs: "check that it found its data.", Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m), Resolved: "Resolved: the data source move on " + m + " is settled"} }), // Modules and providers. kindModuleUnhealthy: worded(func(o conditions.Observation) words { thing := conditions.ThingWords(o) return words{Headline: conditions.Capital(thing) + " is not working", Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)), Resolved: conditions.Capital(thing) + " works again"} }), kindReloginNeeded: worded(func(o conditions.Observation) words { // A module's name may hold dots: it is the id without its machine. module := "" if o.Scope == conditions.ScopeModule && o.Machine != "" { module = strings.TrimSuffix(o.ID, "."+o.Machine) } return reloginWords(orModule(module), machineOr(o, "a machine"), false) }), kindProviderFailing: worded(func(o conditions.Observation) words { thing, consumer := conditions.ThingWords(o), idPart(o, 2) if consumer == "" { consumer = "a module" } return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer, Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+ "it up. It tries again by itself.", conditions.Capital(thing), consumer), Resolved: conditions.Capital(thing) + " serves " + consumer + " again"} }), kindProviderSilent: worded(func(o conditions.Observation) words { thing := conditions.ThingWords(o) return words{Headline: conditions.Capital(thing) + " went quiet", Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+ "reporting.", conditions.Capital(thing)), Resolved: conditions.Capital(thing) + " reports again"} }), kindRetireWaiting: worded(func(o conditions.Observation) words { thing := conditions.ThingWords(o) return words{Headline: "Retiring data on " + thing + " waits for you", Needs: "approve or reject retiring it; the details list what would go.", Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.", conditions.Capital(thing)), Resolved: "Resolved: the retirement on " + thing + " is decided"} }), kindRetireRejected: worded(func(o conditions.Observation) words { thing := conditions.ThingWords(o) return words{Headline: "Unused data kept on " + thing, Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing), Resolved: "Resolved: the unused data on " + thing + " is gone"} }), kindCleanupWaiting: worded(func(o conditions.Observation) words { var where string if o.Scope == conditions.ScopeProvider { where = conditions.ThingWords(o) } else { what, _ := dataWords(o) where = what + " on " + machineOr(o, "a machine") } return words{Headline: "Retired data waits for cleanup", Needs: "delete it or keep it; nothing is deleted without your word.", Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where), Resolved: "Resolved: the retired data is dealt with"} }), // Data. kindDataUnmeasured: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Data on " + m + " could not be measured", Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+ "the data there. The next measurement may settle it.", m), Resolved: "Data on " + m + " is measured again"} }), kindProtectionMissing: worded(func(o conditions.Observation) words { what, _ := dataWords(o) return words{Headline: conditions.Capital(what) + " is unprotected", Needs: "check the storage it is on.", Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+ "the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")), Resolved: conditions.Capital(what) + " is protected again"} }), kindDataMissing: worded(func(o conditions.Observation) words { what, _ := dataWords(o) return words{Headline: conditions.Capital(what) + " is gone", // No silence from a click: data loss is never waved away from a notification (ADR 0258). Needs: "restore it from a backup, or silence this " + FromMeshMCPServer, Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")), Resolved: conditions.Capital(what) + " is back"} }), kindDataShrank: worded(func(o conditions.Observation) words { what, _ := dataWords(o) m := machineOr(o, "its machine") if idPart(o, 1) == "dataset" { what = "a dataset" } return words{Headline: conditions.Capital(what) + " shrank on " + m, // No silence from a click: data loss is never waved away from a notification (ADR 0258). Needs: "restore the last good copy from a backup, or silence this " + FromMeshMCPServer, Explanation: "More than half of what it held is gone within a week.", Resolved: "Resolved: the shrinking on " + m + " is explained"} }), kindDataQuiet: worded(func(o conditions.Observation) words { what, _ := dataWords(o) return words{Headline: conditions.Capital(what) + " stopped changing", Needs: "check that what writes it still runs.", Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+ "usual.", conditions.Capital(what), machineOr(o, "its machine")), Resolved: conditions.Capital(what) + " is written again"} }), kindBackupStale: worded(func(o conditions.Observation) words { what, _ := dataWords(o) m := machineOr(o, "its machine") return words{Headline: "No recent backup of " + what, Needs: fmt.Sprintf("check that backups run on %s.", m), Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+ "changed since would be lost.", conditions.Capital(what), m), Resolved: conditions.Capital(what) + " is backed up again"} }), kindArrayDegraded: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Storage on " + m + " is degraded", Needs: fmt.Sprintf("check the disks of %s.", m), Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.", Resolved: "Storage on " + m + " is whole again"} }), kindEmptyReplacement: worded(func(o conditions.Observation) words { module := idPart(o, 1) if o.Scope == conditions.ScopeProvider { module = idPart(o, 2) } if module == "" { module = "a module" } return words{Headline: conditions.Capital(module) + " runs on empty data", Needs: "decide where its data lives: move the data, or move it back.", Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+ "elsewhere.", conditions.Capital(module), machineOr(o, "its machine")), Resolved: conditions.Capital(module) + " has its data again"} }), kindDataHeldTwice: worded(func(o conditions.Observation) words { consumer := idPart(o, 1) if consumer == "" { consumer = "a module" } return words{Headline: conditions.Capital(consumer) + "'s data is on several machines", Needs: "decide which copy to keep.", Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)), Resolved: conditions.Capital(consumer) + "'s data is in one place again"} }), // Deliveries and walks. "stalled": worded(func(o conditions.Observation) words { if o.Scope == conditions.ScopeDelivery { return words{Headline: "A delivery is held too long", Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o), Resolved: "Resolved: the delivery moves again"} } return words{Headline: "A delivery is stuck halfway", Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.", Resolved: "Resolved: the delivery moves again"} }), kindWalkWaiting: worded(func(o conditions.Observation) words { return words{Headline: "A delivery is waiting to start", Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " + "out) has not let it start yet.", Resolved: "Resolved: the delivery is no longer waiting"} }), "merge-not-acted": worded(func(o conditions.Observation) words { return words{Headline: "A merge was never picked up", Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " + "sent. The mesh catches up on missed merges by itself.", Resolved: "Resolved: the merge was picked up"} }), "ask-lost": worded(func(o conditions.Observation) words { return words{Headline: "A build has no result", Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", Resolved: "Resolved: the build has its result"} }), kindRolledBack: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") module := idPart(o, 0) if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore { module = "a module" } if o.Scope == conditions.ScopeCore { return words{Headline: "A core update was put back on " + m, Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+ "before was put back and runs.", module, m), Resolved: "Resolved: the core on " + m + " is healthy"} } return words{Headline: conditions.Capital(module) + " update put back on " + m, Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+ "before, which runs. The change is not delivered until it is fixed.", module, m), Resolved: "Resolved: " + module + " on " + m + " is settled"} }), kindRollbackFailed: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Putting back an update failed on " + m, Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m), Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m), Resolved: "Resolved: " + m + " runs a good build again"} }), "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), "facts-stale": worded(func(o conditions.Observation) words { return words{Headline: "Merge checks use outdated facts", Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " + "by themselves.", Resolved: "Merge checks use current facts again"} }), // The controller and the core. "controller-deaf": worded(func(o conditions.Observation) words { return words{Headline: "The controller stopped listening", Needs: "restart the controller if this stays, " + FromMeshMCPServer, Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " + "wait. Changes and repairs do not happen until it recovers.", Resolved: "The controller listens again"} }), "self-check-silent": worded(func(o conditions.Observation) words { return words{Headline: "The mesh's self-check stopped", Needs: "restart the controller if this stays, " + FromMeshMCPServer, Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " + "Problems may go unnoticed until it runs again.", Resolved: "The self-check runs again"} }), "watchdogs-silent": worded(func(o conditions.Observation) words { return words{Headline: "The mesh's watchdogs stopped", Needs: "restart the controller if this stays, " + FromMeshMCPServer, Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " + "missed signals are not noticed.", Resolved: "The watchdogs run again"} }), "lease-lost": worded(func(o conditions.Observation) words { return words{Headline: "The controller lost its lease", Needs: "make sure only one controller runs; the details say which machines claim it.", Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " + "not renew it, so two could act at once, or none.", Resolved: "The controller holds its lease again"} }), "lease-split": worded(func(o conditions.Observation) words { return words{Headline: "Two controllers may be acting", Needs: "make sure only one controller runs; the details say which machines claim it.", Explanation: "The lease that lets one controller act at a time does not agree with its record.", Resolved: "One controller acts again"} }), "stale-writer": worded(func(o conditions.Observation) words { return words{Headline: "Outdated instructions are being sent", Explanation: "Something kept sending instructions older than what the machines already hold, and they " + "were refused, which is harmless.", Resolved: "Resolved: no more outdated instructions"} }), "status-slow": worded(func(o conditions.Observation) words { return words{Headline: "The mesh answers slowly", Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.", Resolved: "The mesh answers quickly again"} }), kindCoreUnhealthy: worded(func(o conditions.Observation) words { if o.Scope == conditions.ScopeBus { return words{Headline: "The bus is not healthy", Needs: "check the machine the bus runs on; the details say what fails.", Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " + "and tool calls may fail until it recovers.", Resolved: "The bus is healthy again"} } m := machineOr(o, "its machine") return words{Headline: "The mesh's own software fails on " + m, Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m), Resolved: "The mesh's own software on " + m + " is healthy again"} }), "call-hung": worded(func(o conditions.Observation) words { return words{Headline: "A tool call is hanging", Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.", Resolved: "Resolved: the tool call ended"} }), "probe-failed": worded(func(o conditions.Observation) words { return words{Headline: "A check of the mesh could not run", Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " + "again every few minutes.", Resolved: "The check runs again"} }), kindHealersBraked: worded(func(o conditions.Observation) words { return words{Headline: "All automatic repairs stopped", Needs: "read the details and decide whether the repairs may run again.", Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " + "looks like a loop, so all of them stopped.", Resolved: "Automatic repairs run again"} }), "healer-wanted": worded(func(o conditions.Observation) words { cause := strings.TrimPrefix(o.ID, "hand-acts.") if o.Scope != conditions.ScopeMesh { cause = "something" } return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand", Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " + "broken now.", Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")} }), // The bus and seats. "resolver-wrong": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Name lookups fail on " + m, Needs: fmt.Sprintf("check that %s is on and online.", m), Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+ "machines that rely on it may not find each other.", m), Resolved: "Name lookups work on " + m + " again"} }), "holder-silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") seat := idPart(o, 0) if o.Scope != conditions.ScopeSeat { seat = "a seat" } return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering", Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+ "machine asleep is normal.", seat, m), Resolved: conditions.Capital(seat) + " on " + m + " answers again"} }), kindBusObjectsUnasserted: worded(func(o conditions.Observation) words { return words{Headline: "The bus could not be fully set up", Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " + "again with the next send.", Resolved: "The bus is fully set up again"} }), kindBusMaintenance: worded(func(o conditions.Observation) words { return words{Headline: "The bus is being upgraded", Explanation: "The mesh's message system is being upgraded; some things pause until it is done.", Resolved: "The bus upgrade is done"} }), kindBusUpgradeFailed: worded(func(o conditions.Observation) words { return words{Headline: "The bus upgrade failed", Needs: "decide whether to put the bus back to the version before; the details say how.", Explanation: "The upgrade of the mesh's message system did not end healthy in time.", Resolved: "Resolved: the bus is healthy after its upgrade"} }), kindConsumerBehind: worded(func(o conditions.Observation) words { return words{Headline: "Messages pile up for a listener", Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " + "healer restarts it if it stays.", Resolved: "The listener caught up"} }), "consumer-wrong": worded(func(o conditions.Observation) words { return words{Headline: "A listener on the bus is set up wrong", Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " + "it up again.", Resolved: "The listener is set up right again"} }), "consumer-lost": worded(func(o conditions.Observation) words { return words{Headline: "A listener on the bus is gone", Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", Resolved: "The listener is back"} }), "slow-consumer": worded(func(o conditions.Observation) words { return words{Headline: "A listener on the bus is too slow", Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.", Resolved: "The listener keeps up again"} }), "max-deliveries": worded(func(o conditions.Observation) words { return words{Headline: "A message could not be handled", Explanation: "The bus gave up on a message after trying to hand it over too many times.", Resolved: "Resolved: messages are handled again"} }), "refused": worded(func(o conditions.Observation) words { return words{Headline: "The bus refuses some messages", Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.", Resolved: "Resolved: the bus takes the messages again"} }), "stream-wrong": worded(func(o conditions.Observation) words { return words{Headline: "Part of the bus's storage is wrong", Needs: "check the machine the bus runs on; the details say what is missing.", Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.", Resolved: "The bus's storage is right again"} }), "archives-unheld": worded(func(o conditions.Observation) words { return words{Headline: "Kept builds are about to be cleaned up", Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.", Resolved: "Resolved: every kept build is claimed"} }), "archives-missing": worded(func(o conditions.Observation) words { return words{Headline: "Kept builds are missing", Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.", Resolved: "Resolved: the kept builds are back"} }), } // humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days". func humanDuration(d time.Duration) string { switch { case d < 2*time.Minute: return "a minute" case d < 2*time.Hour: return fmt.Sprintf("%d minutes", int(d.Minutes())) case d < 48*time.Hour: return fmt.Sprintf("%d hours", int(d.Hours())) } return fmt.Sprintf("%d days", int(d.Hours()/24)) } // repoName is a repository as a person says it: its name without its owner. func repoName(repository string) string { if i := strings.LastIndex(repository, "/"); i >= 0 { return repository[i+1:] } return repository } // namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more". func namesWords(names []string, most int) string { switch { case len(names) == 0: return "" case len(names) == 1: return names[0] case len(names) <= most: return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] } return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most) } // planModules are the modules a walk moves, sorted. func planModules(p inventory.Plan) []string { out := make([]string, 0, len(p.Modules)) for m := range p.Modules { out = append(out, m) } sort.Strings(out) return out } // deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else // their count and the repository. func deliveryWhat(modules []string, repository string) string { if len(modules) > 0 && len(modules) <= 2 { return namesWords(modules, 2) } if len(modules) > 2 { return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository)) } return repoName(repository) } // deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery". func deliveryName(modules []string, repository string) string { if len(modules) > 0 && len(modules) <= 2 { return namesWords(modules, 2) + " delivery" } return repoName(repository) + " delivery" } // walkWaitingWords explains a walk waiting for its delivery's word. func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string { what := deliveryWhat(w.modules, w.repository) if severity == conditions.Urgent { return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+ "delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in)) } return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+ "decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.", what, humanDuration(in), humanDuration(waitUrgentAfter)) } // waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. // Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying // where they are given without a channel, and the ask's text drops that (askText). func waitingNeeds(severity conditions.Severity) string { if severity == conditions.Urgent { return "start it, or stop it, " + FromMeshMCPServer } return "" } // waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's // own verbs, approved by the operator (novox/hq ADR 0259). None before the bound. func waitingActions(plan string, severity conditions.Severity) []conditions.Action { if severity != conditions.Urgent || plan == "" { return nil } return []conditions.Action{ {Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, {Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, } } // moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its // account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it. // No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258). func moduleNeeds(node string, rs []inventory.ResourceHealth) string { unit := "" for _, r := range rs { if strings.Contains(r.Reason, "relogin needed") { return reloginNeeds(node) } if r.Kind == link.KindUnit && unit == "" { unit = unitPlainWords(r.Target) } } if unit != "" { // Also asked of the operator (moduleActions); the ask's text drops where (askText). return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer) } return "" } // moduleActions are the answers to a module unhealthy on a machine: restart its failed service there, // approved by the operator (novox/hq ADR 0259) — none when the mesh restarts it, or a new login is what it // waits for. func moduleActions(node string, rs []inventory.ResourceHealth) []conditions.Action { for _, r := range rs { if strings.Contains(r.Reason, "relogin needed") { return nil } } for _, r := range rs { if r.Kind != link.KindUnit || r.Target == "" { continue } scope := "system" if r.Account != "" { scope = "user" } return []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: node, Level: conditions.LevelApprove, Arguments: map[string]string{"unit": r.Target, "scope": scope}}} } return nil } // FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP // server (the glossary's word; "console" is retired): the answer is not an // acknowledgement, so it is given where the operator is known to be the one asking, until answers are // authorised (to-be 46 phases 5 and 6). const FromMeshMCPServer = "from the mesh MCP server; this notification cannot do it." // releaseHeldWords are the plain words of updates held after a walk failed: which modules wait, // on which machines, and where the operator releases them (ADR 0258: a release is not an acknowledgement, so // no notification gives it). Raised with the modules and machines (backlogObservation); the kind's fallback // knows neither. func releaseHeldWords(modules, machines []string) words { what := "Some module updates" headline := "Updates wait for your release" if len(modules) > 0 { what = "Updates of " + namesWords(modules, 3) if h := what + " wait for your release"; len(h) <= conditions.HeadlineMax { headline = h } else if h := fmt.Sprintf("%d module updates wait for your release", len(modules)); len(h) <= conditions.HeadlineMax { headline = h } } where := "" if len(machines) > 0 { where = " on " + namesWords(machines, 4) } return words{Headline: headline, Needs: "release them " + FromMeshMCPServer, Explanation: fmt.Sprintf("%s%s wait for a person to release them, because the last walk failed."+ " They are not delivered until then, and stay held if you leave them.", what, where), Resolved: "Resolved: the held updates are released"} } // reloginNeeds is what an account waiting for its groups needs (ADR 0252). func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node) } // stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it. func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string, actions []conditions.Action) { repository, _, _ := strings.Cut(l.ID, "@") name := repoName(repository) held := l.State if held == "" { held = "held" } long := "too long" if d, err := time.ParseDuration(l.For); err == nil { long = "for " + humanDuration(d) } if o.Resolver == conditions.ResolverOperator { // Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the // router says where each can be answered, so the words do not. release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, Arguments: map[string]string{"id": l.ID, "why": ""}} stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, Arguments: map[string]string{"id": l.ID, "why": ""}} switch held { case "held": needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop} case "ready", "checked": needs = "merge its pull request, or close it." default: needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop} } } return fmt.Sprintf("Delivery of %s %s %s", name, held, long), fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long), fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions } // causeWords is a hand-act's cause as a person says it. func causeWords(cause string) string { return strings.NewReplacer(".", " ", "_", " ").Replace(cause) } // unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)". func unitPlainWords(unit string) string { for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer", ".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} { if name, ok := strings.CutSuffix(unit, suffix); ok { if what == "" { return strings.ReplaceAll(name, ".", " ") } return strings.ReplaceAll(name, ".", " ") + " (" + what + ")" } } return strings.ReplaceAll(unit, ".", " ") } // resourcePlainWords is what is wrong with one of a module's resources, as a person says it. func resourcePlainWords(r inventory.ResourceHealth) string { if r.Kind == link.KindUnit { return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason) } what := "its " + strings.ReplaceAll(r.Kind, "-", " ") switch r.Reason { case "restarting": return what + " keeps restarting" case "down": return what + " is not running" } if r.Check != "" { return what + " fails its " + r.Check + " check" } return what + " is not healthy" } // unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be // systemd's result, such as exit-code, or a sentence carrying it). func unitReasonWords(reason string) string { switch { case strings.Contains(reason, "start-limit"): return "failed too often to be started again" case strings.Contains(reason, "exit-code"): return "stopped with an error" case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"): return "crashed" case strings.Contains(reason, "timeout"): return "timed out" } return "failed" } // orModule is a module's name, or "a module" when none is known. func orModule(m string) string { if m == "" { return "a module" } return m }