Compare commits
6
Commits
0012ea562e
...
51d7bbfdc8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
51d7bbfdc8 | ||
|
|
e3b5c224e8 | ||
|
|
6df30b6d98 | ||
|
|
1a4305213d | ||
|
|
7df72edd2b | ||
|
|
1fce541023 |
@@ -113,6 +113,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -77,6 +77,10 @@ type health int
|
||||
|
||||
const (
|
||||
healthGood health = iota
|
||||
// healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for
|
||||
// one person's new login (personWait). The build did what it should; no bound a machine can meet ends
|
||||
// the wait. It counts as a pass, and the gate carries the reading along in its verdict.
|
||||
healthPerson
|
||||
// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
|
||||
// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
|
||||
// build for something it did not do.
|
||||
@@ -209,7 +213,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
if c.Source == gateProbe || c.Raised.Before(since) {
|
||||
// A wait for a person's new login is the module's reading, not a fault raised since the send: the
|
||||
// gate reads it from the statement below (ADR 0254).
|
||||
if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
@@ -435,13 +441,19 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
for node, moved := range byMachine {
|
||||
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
|
||||
}
|
||||
// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
|
||||
// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
|
||||
// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
|
||||
worst, why := healthGood, ""
|
||||
var failing []string
|
||||
broken := map[string]bool{}
|
||||
reading := map[string]health{}
|
||||
waits := map[string]string{}
|
||||
var modules []string
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if h == healthGood {
|
||||
if h == healthGood || h == healthPerson {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
} else if w.whole != "" {
|
||||
@@ -450,7 +462,16 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
h, said = healthWaiting, w.waiting
|
||||
}
|
||||
}
|
||||
if h != healthGood && !slices.Contains(failing, j.module) {
|
||||
if _, seen := reading[j.module]; !seen {
|
||||
modules = append(modules, j.module)
|
||||
}
|
||||
if h > reading[j.module] {
|
||||
reading[j.module] = h
|
||||
}
|
||||
if h == healthPerson {
|
||||
waits[j.module] = joinSaid(waits[j.module], said)
|
||||
}
|
||||
if h > healthPerson && !slices.Contains(failing, j.module) {
|
||||
failing = append(failing, j.module)
|
||||
}
|
||||
if h == healthBroken {
|
||||
@@ -458,14 +479,45 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
}
|
||||
if h > worst {
|
||||
worst, why = h, said
|
||||
} else if h == worst && h != healthGood && why == "" {
|
||||
} else if h == worst && h > healthPerson && why == "" {
|
||||
why = said
|
||||
}
|
||||
}
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
g.Waits = nil
|
||||
for _, m := range modules {
|
||||
switch {
|
||||
case reading[m] > healthPerson:
|
||||
g.Healthy[m] = 0
|
||||
default:
|
||||
g.Healthy[m]++
|
||||
if w := waits[m]; w != "" {
|
||||
if g.Waits == nil {
|
||||
g.Waits = map[string]string{}
|
||||
}
|
||||
g.Waits[m] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
// passedAlone is every module that passed on its own while the send as a whole did not.
|
||||
passedAlone := func() []string {
|
||||
var out []string
|
||||
if now.Sub(*g.Since) < gateSettle {
|
||||
return nil
|
||||
}
|
||||
for _, m := range modules {
|
||||
if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
||||
g.Failing = failing
|
||||
g.Failing, g.Passing = failing, passedAlone()
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
case worst == healthWaiting:
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
@@ -474,19 +526,48 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
if now.Sub(*g.Since) > gateBound {
|
||||
g.Passing = passedAlone()
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
|
||||
}
|
||||
default:
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second)), now)
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
|
||||
func waitsSaid(waits map[string]string) string {
|
||||
if len(waits) == 0 {
|
||||
return ""
|
||||
}
|
||||
modules := make([]string, 0, len(waits))
|
||||
for m := range waits {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
var said []string
|
||||
for _, m := range modules {
|
||||
said = append(said, waits[m])
|
||||
}
|
||||
return "; and it waits for a person: " + strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
func joinSaid(a, b string) string {
|
||||
switch {
|
||||
case a == "":
|
||||
return b
|
||||
case b == "" || strings.Contains(a, b):
|
||||
return a
|
||||
}
|
||||
return a + "; " + b
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
@@ -959,6 +1040,12 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
}
|
||||
if g.Verdict == "" && len(g.Waits) > 0 {
|
||||
line += waitsSaid(g.Waits)
|
||||
}
|
||||
if len(g.Passing) > 0 {
|
||||
line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ")
|
||||
}
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
|
||||
@@ -90,8 +90,9 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
|
||||
if len(u.Failed) == 0 {
|
||||
return o, false
|
||||
}
|
||||
var names, said []string
|
||||
var names, said, plain []string
|
||||
for _, f := range u.Failed {
|
||||
plain = append(plain, unitPlainWords(f.Unit))
|
||||
where := ""
|
||||
if f.Scope == "user" {
|
||||
where = " (the account's own manager)"
|
||||
@@ -111,6 +112,16 @@ func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Obser
|
||||
"Each is the machine's own: mend or remove it there, or have a module place it",
|
||||
node, len(u.Failed), strings.Join(names, ", "))
|
||||
o.Said = strings.Join(said, "; ")
|
||||
failed := "1 failed service"
|
||||
if len(u.Failed) > 1 {
|
||||
failed = fmt.Sprintf("%d failed services", len(u.Failed))
|
||||
}
|
||||
o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
|
||||
o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
|
||||
o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
|
||||
node, namesWords(plain, 3))
|
||||
o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
|
||||
o.Resolved = "No failed services on " + node + " any more"
|
||||
return o, true
|
||||
}
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
for _, r := range h.Resources {
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs}
|
||||
Check: r.Check, Needs: r.Needs, Account: r.Account}
|
||||
resources = append(resources, kept)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
@@ -135,7 +135,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
standing := map[string]conditions.Condition{}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
|
||||
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded) && c.Subject.Machine == node {
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
@@ -155,6 +155,19 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
||||
// operator, never urgent, cleared on the first statement that no longer says it.
|
||||
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
||||
o := reloginObservation(m, node, said, unhealthy[m])
|
||||
seen[o.Key()] = true
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
@@ -167,6 +180,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
@@ -187,6 +201,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
||||
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
||||
if c.Kind == kindReloginNeeded {
|
||||
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
||||
}
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
}
|
||||
@@ -238,11 +255,35 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
return err
|
||||
}
|
||||
|
||||
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
||||
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
||||
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
||||
// person in again, and a restart of the module's service would start it in the same session.
|
||||
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
||||
w := reloginWords(module, node)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
||||
return o
|
||||
}
|
||||
|
||||
// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
|
||||
// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
|
||||
func reloginWords(module, node string) words {
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
||||
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
||||
"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
var words, said []string
|
||||
var words, said, plain []string
|
||||
needs, actions := moduleNeeds(node, rs)
|
||||
for _, r := range rs {
|
||||
plain = append(plain, resourcePlainWords(r))
|
||||
if r.Kind == link.KindUnit {
|
||||
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
||||
// resource that places it — a package, a file — is evidence.
|
||||
@@ -258,8 +299,14 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
}
|
||||
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
||||
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; ")}
|
||||
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
||||
Said: strings.Join(said, "; "),
|
||||
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
||||
namesWords(plain, 3)),
|
||||
Needs: needs,
|
||||
Actions: actions,
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
@@ -287,9 +334,76 @@ func orNotSaid(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
||||
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
||||
const kindReloginNeeded = "relogin-needed"
|
||||
|
||||
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
||||
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
||||
// excused:
|
||||
//
|
||||
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
||||
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
||||
// running began before;
|
||||
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
||||
// accounts (Account names it): the manager that began before the group and does not hold it.
|
||||
//
|
||||
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
||||
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
||||
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
||||
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
||||
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
||||
waiting := map[string]bool{}
|
||||
var accounts []string
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
|
||||
waiting[accountOf(r)] = true
|
||||
accounts = append(accounts, accountOf(r))
|
||||
}
|
||||
}
|
||||
if len(accounts) == 0 {
|
||||
return "", false
|
||||
}
|
||||
var units []string
|
||||
for _, r := range rs {
|
||||
if r.Module != module || r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
||||
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
||||
units = append(units, r.Target)
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
sort.Strings(accounts)
|
||||
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
||||
"running session began before it was; log out of every session and in again, or reboot", machine, module,
|
||||
strings.Join(accounts, ", "))
|
||||
if len(units) > 0 {
|
||||
sort.Strings(units)
|
||||
said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
||||
func accountOf(r inventory.ResourceHealth) string {
|
||||
if r.Account != "" {
|
||||
return r.Account
|
||||
}
|
||||
return r.Target
|
||||
}
|
||||
|
||||
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
||||
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
||||
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
||||
//
|
||||
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
||||
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
||||
// and not something a machine can meet within a bound.
|
||||
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
if f.healthErr != nil {
|
||||
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
||||
@@ -301,10 +415,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if h.HeardAt.Before(since) {
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
}
|
||||
if waits && r.State == link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateHealthy:
|
||||
case link.StateStarting:
|
||||
@@ -320,6 +438,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
if waits {
|
||||
return healthPerson, wait
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254,
|
||||
// issue 318).
|
||||
|
||||
func relogin(module, account string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account,
|
||||
Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module +
|
||||
", and its running session began before it was; log out of every session and in again, or reboot"}
|
||||
}
|
||||
|
||||
func userUnit(module, account string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit,
|
||||
Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
|
||||
Reason: "failed in the account's own service manager (exit-code)"}
|
||||
}
|
||||
|
||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||
// before, so that a fault is never excused.
|
||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
notInGroup := relogin("lights", "operator")
|
||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||
systemUnit := userUnit("lights", "operator")
|
||||
systemUnit.Account = ""
|
||||
otherAccount := userUnit("lights", "guest")
|
||||
container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
|
||||
State: link.StateUnhealthy, Reason: "down"}
|
||||
otherModule := userUnit("sound", "operator")
|
||||
starting := userUnit("lights", "operator")
|
||||
starting.State = link.StateStarting
|
||||
byTarget := relogin("lights", "operator")
|
||||
byTarget.Account = "" // an account said by its target alone is still that account
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
rs []inventory.ResourceHealth
|
||||
waits bool
|
||||
}{
|
||||
{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
|
||||
{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
|
||||
userUnit("lights", "operator")}, true},
|
||||
{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
|
||||
{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
|
||||
{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
|
||||
{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
|
||||
{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
|
||||
{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
|
||||
{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
|
||||
{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
|
||||
} {
|
||||
said, waits := personWait("lights", "laptop", c.rs)
|
||||
if waits != c.waits {
|
||||
t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
|
||||
continue
|
||||
}
|
||||
if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
|
||||
t.Errorf("%s: said %q", c.name, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||
// with a container down beside it is not yet healthy, as before.
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
|
||||
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
h := f.health["laptop"]
|
||||
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
|
||||
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
|
||||
f.health["laptop"] = h
|
||||
if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet {
|
||||
t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
at := h0
|
||||
say := func(rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
asLink := func(r inventory.ResourceHealth) link.ResourceHealth {
|
||||
return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State,
|
||||
Reason: r.Reason, Account: r.Account}
|
||||
}
|
||||
account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator"))
|
||||
openNow := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
say(account, unit)
|
||||
say(account, unit)
|
||||
list := openNow()
|
||||
if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" {
|
||||
t.Fatalf("raised %+v; want lights' relogin-needed alone", list)
|
||||
}
|
||||
c := list[0]
|
||||
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator ||
|
||||
!strings.HasPrefix(c.Summary, "relogin needed on laptop:") {
|
||||
t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary)
|
||||
}
|
||||
// After the new login the account is healthy, and the unit, still failing, is the module's own fault.
|
||||
account.State, account.Reason = link.StateHealthy, ""
|
||||
say(account, unit)
|
||||
list = openNow()
|
||||
if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" {
|
||||
t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list)
|
||||
}
|
||||
unit.State, unit.Reason = link.StateHealthy, ""
|
||||
say(account, unit)
|
||||
if list = openNow(); len(list) != 0 {
|
||||
t.Fatalf("after the unit runs: %+v; want nothing open", list)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||
Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy,
|
||||
Reason: "down"}}}}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateBound = 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
||||
!strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed {
|
||||
t.Fatal("late's build is not marked failed at its gate")
|
||||
}
|
||||
current, _ := inv.CurrentBuilds(ctx)
|
||||
if current["app"].Commit != "c2" || current["late"].Commit != "c1" {
|
||||
t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit,
|
||||
current["late"].Commit)
|
||||
}
|
||||
if !strings.Contains(p.Note, "passed on their own and kept: app") {
|
||||
t.Errorf("the walk does not say what kept its pass: %s", p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||
// no button, since nothing but their own new login can do it.
|
||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
|
||||
"group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+
|
||||
"update is in place and nothing was undone.")
|
||||
if len(o.Actions) != 0 || o.Needs == "" {
|
||||
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
|
||||
}
|
||||
// The kind's own wording, for a condition raised without words, says the same.
|
||||
if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs ||
|
||||
w.Headline != o.Headline || len(w.Actions) != 0 {
|
||||
t.Errorf("the kind's wording: %+v", w)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,776 @@
|
||||
package main
|
||||
|
||||
// The plain words of every condition kind the controller raises (novox/hq ADR 0253): what the operator
|
||||
// reads — a headline, one or two sentences on what it means and whether to act, and the line said when it
|
||||
// clears. The summary beside them keeps the ids, commits and verbs for whoever looks closer.
|
||||
//
|
||||
// A kind whose words need more than the condition's subject (a walk's modules, a unit's name, a hand-act's
|
||||
// cause) is worded where it is raised, and its kind here is the fallback. Every kind raised in the test
|
||||
// suite is held to having words, and to their being plain (plain_words_test.go).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
type words = conditions.Words
|
||||
|
||||
// worded is a wording that needs only the condition's subject.
|
||||
func worded(fn func(o conditions.Observation) words) func(conditions.Observation) words { return fn }
|
||||
|
||||
func init() {
|
||||
for kind, fn := range plainWordings {
|
||||
conditions.Wording(kind, fn)
|
||||
}
|
||||
}
|
||||
|
||||
// machineOr is the machine a condition concerns, or what to say when none is named.
|
||||
func machineOr(o conditions.Observation, none string) string {
|
||||
if o.Machine != "" {
|
||||
return o.Machine
|
||||
}
|
||||
return none
|
||||
}
|
||||
|
||||
// idPart is the n-th dotted part of a condition's id, or "".
|
||||
func idPart(o conditions.Observation, n int) string {
|
||||
parts := strings.Split(o.ID, ".")
|
||||
if n < len(parts) {
|
||||
return parts[n]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dataWords is "<item> of <module>" for a data condition keyed by machine, module and item.
|
||||
func dataWords(o conditions.Observation) (what, module string) {
|
||||
module, item := idPart(o, 1), strings.Join(strings.Split(o.ID, ".")[min(2, len(strings.Split(o.ID, "."))):], " ")
|
||||
switch {
|
||||
case module != "" && item != "":
|
||||
return item + " of " + module, module
|
||||
case module != "":
|
||||
return "data of " + module, module
|
||||
}
|
||||
return "data", "a module"
|
||||
}
|
||||
|
||||
var plainWordings = map[string]func(conditions.Observation) words{
|
||||
// Machines and their node-engine.
|
||||
"silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
w := words{Headline: m + " is not answering",
|
||||
Explanation: fmt.Sprintf("The mesh has not heard from %s for several minutes. A machine asleep or away is "+
|
||||
"normal.", m),
|
||||
Resolved: m + " answers again"}
|
||||
if o.Severity == conditions.Urgent {
|
||||
w.Needs = fmt.Sprintf("check that %s is on and online.", m)
|
||||
w.Explanation = fmt.Sprintf("The mesh has not heard from %s for half an hour, and the mesh is run from it: "+
|
||||
"nothing changes anywhere until it is back.", m)
|
||||
}
|
||||
return w
|
||||
}),
|
||||
"sent-not-reported": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has not confirmed a change",
|
||||
Explanation: fmt.Sprintf("The mesh sent %s new instructions and %s has not said it applied them yet. "+
|
||||
"It usually catches up by itself.", m, m),
|
||||
Resolved: m + " applied the change"}
|
||||
}),
|
||||
"tools-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Tools on " + m + " are not answering",
|
||||
Explanation: fmt.Sprintf("The tool runner on %s has not checked in, so nothing can be done on %s through "+
|
||||
"the mesh until it does. A machine asleep is normal.", m, m),
|
||||
Resolved: "Tools on " + m + " answer again"}
|
||||
}),
|
||||
kindAwaitingPush: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " waits for new instructions",
|
||||
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
|
||||
conditions.Capital(m)),
|
||||
Resolved: m + " got its new instructions"}
|
||||
}),
|
||||
"declaration-refused": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeMesh {
|
||||
return words{Headline: "No machine can get new instructions",
|
||||
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
||||
Explanation: "The mesh cannot work out its private network, so no machine's instructions can be " +
|
||||
"made. Machines keep running what they have.",
|
||||
Resolved: "Machines can get new instructions again"}
|
||||
}
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot get new instructions",
|
||||
Needs: "decide whether to undo the last catalogue change; the details say what fails.",
|
||||
Explanation: fmt.Sprintf("What the mesh would send %s is wrong and would be refused, so nothing new "+
|
||||
"reaches it. It keeps running what it has.", m),
|
||||
Resolved: m + " can get new instructions again"}
|
||||
}),
|
||||
"own-address-banned": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has banned the mesh",
|
||||
Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", m),
|
||||
Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+
|
||||
"mesh is locked out of it.", m),
|
||||
Resolved: m + " no longer bans the mesh"}
|
||||
}),
|
||||
"core-behind": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " runs an old node-engine",
|
||||
Explanation: fmt.Sprintf("%s runs an older build of the mesh's own software than the mesh holds. It "+
|
||||
"works, and gets the newer one with a later delivery.", conditions.Capital(m)),
|
||||
Resolved: m + " runs the current node-engine"}
|
||||
}),
|
||||
kindMachineUnits: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Failed services on " + m,
|
||||
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", m),
|
||||
Explanation: "No module manages them, so the mesh does not repair them.",
|
||||
Resolved: "No failed services on " + m + " any more", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
||||
}),
|
||||
kindMachineNetwork: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " has a network problem",
|
||||
Explanation: fmt.Sprintf("%s's own network check failed: its name lookups, tunnel, bus connection or "+
|
||||
"route. It often passes again once a network change settles.", conditions.Capital(m)),
|
||||
Resolved: m + "'s network is fine again"}
|
||||
}),
|
||||
kindNetworkRewritten: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + "'s network settings were changed",
|
||||
Explanation: fmt.Sprintf("Something other than the mesh rewrote %s's network settings, often a VPN "+
|
||||
"client. Mesh names may not resolve there until it is undone.", m),
|
||||
Resolved: m + "'s network settings are the mesh's again"}
|
||||
}),
|
||||
kindNetworkUnreachable: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " cannot be reached",
|
||||
Explanation: fmt.Sprintf("Other machines cannot reach %s over the mesh's network. A machine away or "+
|
||||
"asleep is normal.", m),
|
||||
Resolved: m + " can be reached again"}
|
||||
}),
|
||||
kindBindingKept: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source is held on " + m,
|
||||
Needs: "decide where its data lives; the details name both places.",
|
||||
Explanation: fmt.Sprintf("A module on %s should switch to another provider of its data, and the mesh "+
|
||||
"kept the old one because switching would leave the data behind.", m),
|
||||
Resolved: "Resolved: the data source on " + m + " is decided"}
|
||||
}),
|
||||
kindBindingMoving: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moves on " + m,
|
||||
Needs: "move its data to the new provider before the next push.",
|
||||
Explanation: fmt.Sprintf("At the next push a module on %s switches to another provider of its data.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
kindBindingMoved: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source moved on " + m,
|
||||
Needs: "check that it found its data.",
|
||||
Explanation: fmt.Sprintf("A module on %s switched to another provider, and its data may still be with the old one.", m),
|
||||
Resolved: "Resolved: the data source move on " + m + " is settled"}
|
||||
}),
|
||||
|
||||
// Modules and providers.
|
||||
kindModuleUnhealthy: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " is not working",
|
||||
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " works again"}
|
||||
}),
|
||||
kindReloginNeeded: worded(func(o conditions.Observation) words {
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule {
|
||||
module = idPart(o, 0)
|
||||
}
|
||||
return reloginWords(orModule(module), machineOr(o, "a machine"))
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(thing) + " keeps failing " + consumer,
|
||||
Explanation: fmt.Sprintf("%s provides something %s needs, such as a database, and keeps failing to set "+
|
||||
"it up. It tries again by itself.", conditions.Capital(thing), consumer),
|
||||
Resolved: conditions.Capital(thing) + " serves " + consumer + " again"}
|
||||
}),
|
||||
kindProviderSilent: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: conditions.Capital(thing) + " went quiet",
|
||||
Explanation: fmt.Sprintf("%s said it keeps failing a module that depends on it, and then stopped "+
|
||||
"reporting.", conditions.Capital(thing)),
|
||||
Resolved: conditions.Capital(thing) + " reports again"}
|
||||
}),
|
||||
kindRetireWaiting: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Retiring data on " + thing + " waits for you",
|
||||
Needs: "approve or reject retiring it; the details list what would go.",
|
||||
Explanation: fmt.Sprintf("%s keeps data the mesh no longer asks for, too much to retire on its own.",
|
||||
conditions.Capital(thing)),
|
||||
Resolved: "Resolved: the retirement on " + thing + " is decided"}
|
||||
}),
|
||||
kindRetireRejected: worded(func(o conditions.Observation) words {
|
||||
thing := conditions.ThingWords(o)
|
||||
return words{Headline: "Unused data kept on " + thing,
|
||||
Explanation: fmt.Sprintf("You chose to keep data on %s that the mesh no longer asks for; it is still there.", thing),
|
||||
Resolved: "Resolved: the unused data on " + thing + " is gone"}
|
||||
}),
|
||||
kindCleanupWaiting: worded(func(o conditions.Observation) words {
|
||||
var where string
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
where = conditions.ThingWords(o)
|
||||
} else {
|
||||
what, _ := dataWords(o)
|
||||
where = what + " on " + machineOr(o, "a machine")
|
||||
}
|
||||
return words{Headline: "Retired data waits for cleanup",
|
||||
Needs: "delete it or keep it; nothing is deleted without your word.",
|
||||
Explanation: fmt.Sprintf("Retired data of %s has been kept for over thirty days.", where),
|
||||
Resolved: "Resolved: the retired data is dealt with"}
|
||||
}),
|
||||
|
||||
// Data.
|
||||
kindDataUnmeasured: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Data on " + m + " could not be measured",
|
||||
Explanation: fmt.Sprintf("The backup holder on %s did not say what it measured, so nothing is known about "+
|
||||
"the data there. The next measurement may settle it.", m),
|
||||
Resolved: "Data on " + m + " is measured again"}
|
||||
}),
|
||||
kindProtectionMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is unprotected",
|
||||
Needs: "check the storage it is on.",
|
||||
Explanation: fmt.Sprintf("%s on %s is said to be protected by redundant storage, and it is not on any "+
|
||||
"the mesh can see.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is protected again"}
|
||||
}),
|
||||
kindDataMissing: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " is gone",
|
||||
Needs: "restore it from a backup, or silence this if you removed it.",
|
||||
Explanation: fmt.Sprintf("Where %s is kept on %s, nothing exists any more.", what, machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is back", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
||||
}),
|
||||
kindDataShrank: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
m := machineOr(o, "its machine")
|
||||
if idPart(o, 1) == "dataset" {
|
||||
what = "a dataset"
|
||||
}
|
||||
return words{Headline: conditions.Capital(what) + " shrank on " + m,
|
||||
Needs: "if you meant it, silence this; if not, restore the last good copy from a backup.",
|
||||
Explanation: "More than half of what it held is gone within a week.",
|
||||
Resolved: "Resolved: the shrinking on " + m + " is explained", Actions: []conditions.Action{conditions.SilenceAction(o.Key())}}
|
||||
}),
|
||||
kindDataQuiet: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
return words{Headline: conditions.Capital(what) + " stopped changing",
|
||||
Needs: "check that what writes it still runs.",
|
||||
Explanation: fmt.Sprintf("%s on %s is normally written all the time and has not been for longer than "+
|
||||
"usual.", conditions.Capital(what), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(what) + " is written again"}
|
||||
}),
|
||||
kindBackupStale: worded(func(o conditions.Observation) words {
|
||||
what, _ := dataWords(o)
|
||||
m := machineOr(o, "its machine")
|
||||
return words{Headline: "No recent backup of " + what,
|
||||
Needs: fmt.Sprintf("check that backups run on %s.", m),
|
||||
Explanation: fmt.Sprintf("%s on %s has no good backup recent enough. If the machine failed now, what "+
|
||||
"changed since would be lost.", conditions.Capital(what), m),
|
||||
Resolved: conditions.Capital(what) + " is backed up again"}
|
||||
}),
|
||||
kindArrayDegraded: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Storage on " + m + " is degraded",
|
||||
Needs: fmt.Sprintf("check the disks of %s.", m),
|
||||
Explanation: "Its redundant storage lost a disk or is rebuilding; the data on it is less protected until it is whole.",
|
||||
Resolved: "Storage on " + m + " is whole again"}
|
||||
}),
|
||||
kindEmptyReplacement: worded(func(o conditions.Observation) words {
|
||||
module := idPart(o, 1)
|
||||
if o.Scope == conditions.ScopeProvider {
|
||||
module = idPart(o, 2)
|
||||
}
|
||||
if module == "" {
|
||||
module = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " runs on empty data",
|
||||
Needs: "decide where its data lives: move the data, or move it back.",
|
||||
Explanation: fmt.Sprintf("%s on %s is using an empty copy of its data while the full copy is kept "+
|
||||
"elsewhere.", conditions.Capital(module), machineOr(o, "its machine")),
|
||||
Resolved: conditions.Capital(module) + " has its data again"}
|
||||
}),
|
||||
kindDataHeldTwice: worded(func(o conditions.Observation) words {
|
||||
consumer := idPart(o, 1)
|
||||
if consumer == "" {
|
||||
consumer = "a module"
|
||||
}
|
||||
return words{Headline: conditions.Capital(consumer) + "'s data is on several machines",
|
||||
Needs: "decide which copy to keep.",
|
||||
Explanation: fmt.Sprintf("%s has live data on more than one machine and uses only one.", conditions.Capital(consumer)),
|
||||
Resolved: conditions.Capital(consumer) + "'s data is in one place again"}
|
||||
}),
|
||||
|
||||
// Deliveries and walks.
|
||||
"stalled": worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeDelivery {
|
||||
return words{Headline: "A delivery is held too long",
|
||||
Explanation: "A delivery has been held past its time limit.", Needs: conditions.ResolverNeeds(o),
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}
|
||||
return words{Headline: "A delivery is stuck halfway",
|
||||
Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.",
|
||||
Resolved: "Resolved: the delivery moves again"}
|
||||
}),
|
||||
kindWalkWaiting: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery is waiting to start",
|
||||
Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " +
|
||||
"out) has not let it start yet.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
"merge-not-acted": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A merge was never picked up",
|
||||
Explanation: "A pull request was merged, and the controller never heard of it, so nothing was built or " +
|
||||
"sent. The mesh catches up on missed merges by itself.",
|
||||
Resolved: "Resolved: the merge was picked up"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||
Resolved: "Resolved: the build has its result"}
|
||||
}),
|
||||
kindRolledBack: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
module := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore {
|
||||
module = "a module"
|
||||
}
|
||||
if o.Scope == conditions.ScopeCore {
|
||||
return words{Headline: "A core update was put back on " + m,
|
||||
Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+
|
||||
"before was put back and runs.", module, m),
|
||||
Resolved: "Resolved: the core on " + m + " is healthy"}
|
||||
}
|
||||
return words{Headline: conditions.Capital(module) + " update put back on " + m,
|
||||
Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+
|
||||
"before, which runs. The change is not delivered until it is fixed.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
kindRollbackFailed: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Putting back an update failed on " + m,
|
||||
Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m),
|
||||
Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m),
|
||||
Resolved: "Resolved: " + m + " runs a good build again"}
|
||||
}),
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Updates wait for your release",
|
||||
Needs: "release them, or leave them held.",
|
||||
Explanation: "Some module updates wait for a person to release them, and are not delivered until then.",
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||
"by themselves.",
|
||||
Resolved: "Merge checks use current facts again"}
|
||||
}),
|
||||
|
||||
// The controller and the core.
|
||||
"controller-deaf": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller stopped listening",
|
||||
Needs: "restart the controller if this stays.",
|
||||
Explanation: "The controller, which coordinates the mesh, has taken no messages for minutes while some " +
|
||||
"wait. Changes and repairs do not happen until it recovers.",
|
||||
Resolved: "The controller listens again"}
|
||||
}),
|
||||
"self-check-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's self-check stopped",
|
||||
Needs: "restart the controller if this stays.",
|
||||
Explanation: "The self-check, which looks over the whole mesh every few minutes, has not finished a run. " +
|
||||
"Problems may go unnoticed until it runs again.",
|
||||
Resolved: "The self-check runs again"}
|
||||
}),
|
||||
"watchdogs-silent": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh's watchdogs stopped",
|
||||
Needs: "restart the controller if this stays.",
|
||||
Explanation: "The watchdogs, which notice when something expected does not happen, have not run, so " +
|
||||
"missed signals are not noticed.",
|
||||
Resolved: "The watchdogs run again"}
|
||||
}),
|
||||
"lease-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The controller lost its lease",
|
||||
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
||||
Explanation: "The controller holds a lease so that only one copy of it acts at a time. It lost or could " +
|
||||
"not renew it, so two could act at once, or none.",
|
||||
Resolved: "The controller holds its lease again"}
|
||||
}),
|
||||
"lease-split": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Two controllers may be acting",
|
||||
Needs: "make sure only one controller runs; the details say which machines claim it.",
|
||||
Explanation: "The lease that lets one controller act at a time does not agree with its record.",
|
||||
Resolved: "One controller acts again"}
|
||||
}),
|
||||
"stale-writer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Outdated instructions are being sent",
|
||||
Explanation: "Something kept sending instructions older than what the machines already hold, and they " +
|
||||
"were refused, which is harmless.",
|
||||
Resolved: "Resolved: no more outdated instructions"}
|
||||
}),
|
||||
"status-slow": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The mesh answers slowly",
|
||||
Explanation: "Asking the mesh for its status took longer than ten seconds. It still works, only slowly.",
|
||||
Resolved: "The mesh answers quickly again"}
|
||||
}),
|
||||
kindCoreUnhealthy: worded(func(o conditions.Observation) words {
|
||||
if o.Scope == conditions.ScopeBus {
|
||||
return words{Headline: "The bus is not healthy",
|
||||
Needs: "check the machine the bus runs on; the details say what fails.",
|
||||
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds " +
|
||||
"and tool calls may fail until it recovers.",
|
||||
Resolved: "The bus is healthy again"}
|
||||
}
|
||||
m := machineOr(o, "its machine")
|
||||
return words{Headline: "The mesh's own software fails on " + m,
|
||||
Explanation: fmt.Sprintf("A part of the mesh's own software on %s is not healthy; it is restarted by itself.", m),
|
||||
Resolved: "The mesh's own software on " + m + " is healthy again"}
|
||||
}),
|
||||
"call-hung": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A tool call is hanging",
|
||||
Explanation: "A call to one of the mesh's tools has run far past its time limit; it is ended by itself.",
|
||||
Resolved: "Resolved: the tool call ended"}
|
||||
}),
|
||||
"probe-failed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A check of the mesh could not run",
|
||||
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs " +
|
||||
"again every few minutes.",
|
||||
Resolved: "The check runs again"}
|
||||
}),
|
||||
kindHealersBraked: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "All automatic repairs stopped",
|
||||
Needs: "read the details and decide whether the repairs may run again.",
|
||||
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which " +
|
||||
"looks like a loop, so all of them stopped.",
|
||||
Resolved: "Automatic repairs run again"}
|
||||
}),
|
||||
"healer-wanted": worded(func(o conditions.Observation) words {
|
||||
cause := strings.TrimPrefix(o.ID, "hand-acts.")
|
||||
if o.Scope != conditions.ScopeMesh {
|
||||
cause = "something"
|
||||
}
|
||||
return words{Headline: conditions.Capital(strings.ReplaceAll(cause, ".", " ")) + " keeps being fixed by hand",
|
||||
Explanation: "A person keeps repairing this by hand, and an automatic repair is wanted for it. Nothing is " +
|
||||
"broken now.",
|
||||
Resolved: "Resolved: no more hand repairs of " + strings.ReplaceAll(cause, ".", " ")}
|
||||
}),
|
||||
|
||||
// The bus and seats.
|
||||
"resolver-wrong": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Name lookups fail on " + m,
|
||||
Needs: fmt.Sprintf("check that %s is on and online.", m),
|
||||
Explanation: fmt.Sprintf("The mesh's name resolver on %s does not answer machine names correctly, so "+
|
||||
"machines that rely on it may not find each other.", m),
|
||||
Resolved: "Name lookups work on " + m + " again"}
|
||||
}),
|
||||
"holder-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
seat := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeSeat {
|
||||
seat = "a seat"
|
||||
}
|
||||
return words{Headline: conditions.Capital(seat) + " on " + m + " is not answering",
|
||||
Explanation: fmt.Sprintf("Whatever serves %s on %s does not answer, so its tools do nothing there. A "+
|
||||
"machine asleep is normal.", seat, m),
|
||||
Resolved: conditions.Capital(seat) + " on " + m + " answers again"}
|
||||
}),
|
||||
kindBusObjectsUnasserted: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus could not be fully set up",
|
||||
Explanation: "When instructions were sent, the bus's queues could not all be checked; they are checked " +
|
||||
"again with the next send.",
|
||||
Resolved: "The bus is fully set up again"}
|
||||
}),
|
||||
kindBusMaintenance: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus is being upgraded",
|
||||
Explanation: "The mesh's message system is being upgraded; some things pause until it is done.",
|
||||
Resolved: "The bus upgrade is done"}
|
||||
}),
|
||||
kindBusUpgradeFailed: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus upgrade failed",
|
||||
Needs: "decide whether to put the bus back to the version before; the details say how.",
|
||||
Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
|
||||
Resolved: "Resolved: the bus is healthy after its upgrade"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
"healer restarts it if it stays.",
|
||||
Resolved: "The listener caught up"}
|
||||
}),
|
||||
"consumer-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is set up wrong",
|
||||
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets " +
|
||||
"it up again.",
|
||||
Resolved: "The listener is set up right again"}
|
||||
}),
|
||||
"consumer-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is gone",
|
||||
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
||||
Resolved: "The listener is back"}
|
||||
}),
|
||||
"slow-consumer": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A listener on the bus is too slow",
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
"max-deliveries": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A message could not be handled",
|
||||
Explanation: "The bus gave up on a message after trying to hand it over too many times.",
|
||||
Resolved: "Resolved: messages are handled again"}
|
||||
}),
|
||||
"refused": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus refuses some messages",
|
||||
Explanation: "The bus refused messages from a part of the mesh, so what they carried did not happen.",
|
||||
Resolved: "Resolved: the bus takes the messages again"}
|
||||
}),
|
||||
"stream-wrong": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
Explanation: "A store the controller keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
||||
Resolved: "The bus's storage is right again"}
|
||||
}),
|
||||
"archives-unheld": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are about to be cleaned up",
|
||||
Explanation: "Some kept builds are not claimed by any module, so the store's cleanup would delete them.",
|
||||
Resolved: "Resolved: every kept build is claimed"}
|
||||
}),
|
||||
"archives-missing": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Kept builds are missing",
|
||||
Explanation: "Some builds the mesh relies on are no longer in the store; they are built again when needed.",
|
||||
Resolved: "Resolved: the kept builds are back"}
|
||||
}),
|
||||
}
|
||||
|
||||
// humanDuration is a duration as a person says it: "56 minutes", "36 hours", "3 days".
|
||||
func humanDuration(d time.Duration) string {
|
||||
switch {
|
||||
case d < 2*time.Minute:
|
||||
return "a minute"
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
// repoName is a repository as a person says it: its name without its owner.
|
||||
func repoName(repository string) string {
|
||||
if i := strings.LastIndex(repository, "/"); i >= 0 {
|
||||
return repository[i+1:]
|
||||
}
|
||||
return repository
|
||||
}
|
||||
|
||||
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
|
||||
func namesWords(names []string, most int) string {
|
||||
switch {
|
||||
case len(names) == 0:
|
||||
return ""
|
||||
case len(names) == 1:
|
||||
return names[0]
|
||||
case len(names) <= most:
|
||||
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||
}
|
||||
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
||||
}
|
||||
|
||||
// planModules are the modules a walk moves, sorted.
|
||||
func planModules(p inventory.Plan) []string {
|
||||
out := make([]string, 0, len(p.Modules))
|
||||
for m := range p.Modules {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
||||
// their count and the repository.
|
||||
func deliveryWhat(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2)
|
||||
}
|
||||
if len(modules) > 2 {
|
||||
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
|
||||
}
|
||||
return repoName(repository)
|
||||
}
|
||||
|
||||
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
|
||||
func deliveryName(modules []string, repository string) string {
|
||||
if len(modules) > 0 && len(modules) <= 2 {
|
||||
return namesWords(modules, 2) + " delivery"
|
||||
}
|
||||
return repoName(repository) + " delivery"
|
||||
}
|
||||
|
||||
// walkWaitingWords explains a walk waiting for its delivery's word.
|
||||
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
|
||||
what := deliveryWhat(w.modules, w.repository)
|
||||
if severity == conditions.Urgent {
|
||||
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
|
||||
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
|
||||
}
|
||||
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
|
||||
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
|
||||
what, humanDuration(in), humanDuration(waitUrgentAfter))
|
||||
}
|
||||
|
||||
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
||||
func waitingNeeds(severity conditions.Severity) string {
|
||||
if severity == conditions.Urgent {
|
||||
return "start it, or stop it."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
|
||||
// the module that should have said go is the one not answering.
|
||||
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
|
||||
if severity != conditions.Urgent {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Action{
|
||||
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
|
||||
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
|
||||
}
|
||||
}
|
||||
|
||||
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
||||
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
||||
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
|
||||
var actions []conditions.Action
|
||||
for _, r := range rs {
|
||||
if strings.Contains(r.Reason, "relogin needed") {
|
||||
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
|
||||
}
|
||||
if r.Kind == link.KindUnit && len(actions) < 2 {
|
||||
scope := "system"
|
||||
if strings.Contains(r.Reason, "account's own") {
|
||||
scope = "user"
|
||||
}
|
||||
label := "Restart"
|
||||
if len(actions) > 0 {
|
||||
label = "Restart " + unitPlainWords(r.Target)
|
||||
}
|
||||
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
|
||||
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
|
||||
}
|
||||
}
|
||||
if len(actions) > 0 {
|
||||
return "restart it; if it fails again, the details say why.", actions
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
||||
func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
|
||||
actions []conditions.Action) {
|
||||
repository, _, _ := strings.Cut(l.ID, "@")
|
||||
name := repoName(repository)
|
||||
held := l.State
|
||||
if held == "" {
|
||||
held = "held"
|
||||
}
|
||||
long := "too long"
|
||||
if d, err := time.ParseDuration(l.For); err == nil {
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
switch held {
|
||||
case "held":
|
||||
needs = "release it, or stop it."
|
||||
actions = []conditions.Action{
|
||||
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
|
||||
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
|
||||
}
|
||||
case "ready", "checked":
|
||||
needs = "merge its pull request, or close it."
|
||||
default:
|
||||
needs = "stop it, or read the details to see what it waits for."
|
||||
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long),
|
||||
fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions
|
||||
}
|
||||
|
||||
// causeWords is a hand-act's cause as a person says it.
|
||||
func causeWords(cause string) string {
|
||||
return strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
}
|
||||
|
||||
// unitPlainWords is a unit as a person names it: "greenclip", "storage-media (a mount)".
|
||||
func unitPlainWords(unit string) string {
|
||||
for suffix, what := range map[string]string{".service": "", ".mount": "a mount", ".timer": "a timer",
|
||||
".socket": "a socket", ".path": "a path unit", ".automount": "an automount", ".scope": "a scope"} {
|
||||
if name, ok := strings.CutSuffix(unit, suffix); ok {
|
||||
if what == "" {
|
||||
return strings.ReplaceAll(name, ".", " ")
|
||||
}
|
||||
return strings.ReplaceAll(name, ".", " ") + " (" + what + ")"
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(unit, ".", " ")
|
||||
}
|
||||
|
||||
// resourcePlainWords is what is wrong with one of a module's resources, as a person says it.
|
||||
func resourcePlainWords(r inventory.ResourceHealth) string {
|
||||
if r.Kind == link.KindUnit {
|
||||
return "its service " + unitPlainWords(r.Target) + " " + unitReasonWords(r.Reason)
|
||||
}
|
||||
what := "its " + strings.ReplaceAll(r.Kind, "-", " ")
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return what + " keeps restarting"
|
||||
case "down":
|
||||
return what + " is not running"
|
||||
}
|
||||
if r.Check != "" {
|
||||
return what + " fails its " + r.Check + " check"
|
||||
}
|
||||
return what + " is not healthy"
|
||||
}
|
||||
|
||||
// unitReasonWords is how a unit failed, as a person says it, from the node-engine's words (which may be
|
||||
// systemd's result, such as exit-code, or a sentence carrying it).
|
||||
func unitReasonWords(reason string) string {
|
||||
switch {
|
||||
case strings.Contains(reason, "start-limit"):
|
||||
return "failed too often to be started again"
|
||||
case strings.Contains(reason, "exit-code"):
|
||||
return "stopped with an error"
|
||||
case strings.Contains(reason, "core-dump"), strings.Contains(reason, "signal"):
|
||||
return "crashed"
|
||||
case strings.Contains(reason, "timeout"):
|
||||
return "timed out"
|
||||
}
|
||||
return "failed"
|
||||
}
|
||||
|
||||
// orModule is a module's name, or "a module" when none is known.
|
||||
func orModule(m string) string {
|
||||
if m == "" {
|
||||
return "a module"
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The conditions the operator read on 2026-10-08, raised again from the same facts (novox/hq ADR 0253):
|
||||
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
|
||||
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
|
||||
|
||||
// plainExample checks one finding's words as the keeper will keep them: plain, within their bounds,
|
||||
// opened by the verdict, and saying what the operator needs; and its actions by label.
|
||||
func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) {
|
||||
t.Helper()
|
||||
machines := append([]string{o.Machine}, o.Also...)
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
||||
Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
||||
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
|
||||
}
|
||||
said := conditions.Verdict(o.Needs, o.Explanation)
|
||||
if o.Headline != headline || said != explanation {
|
||||
t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation)
|
||||
}
|
||||
var labels []string
|
||||
for _, a := range o.Actions {
|
||||
labels = append(labels, a.Label)
|
||||
}
|
||||
if strings.Join(labels, ",") != strings.Join(actions, ",") {
|
||||
t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions)
|
||||
}
|
||||
quiet := ""
|
||||
if o.Needs == "" && o.Severity == conditions.Warning {
|
||||
quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)"
|
||||
}
|
||||
t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet,
|
||||
said, labels, o.Resolved)
|
||||
}
|
||||
|
||||
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
|
||||
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
|
||||
// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no
|
||||
// popup; past them, the operator starts or stops it from the notification.
|
||||
func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
|
||||
awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+
|
||||
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+
|
||||
"for you if it still waits after 4 hours.")
|
||||
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
|
||||
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
|
||||
}
|
||||
|
||||
// Past four hours it is urgent, and offers the controller's own answers.
|
||||
f.waits[0].since = now.Add(-5 * time.Hour)
|
||||
got = watchWaits(f)
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
||||
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
|
||||
"be stuck.", "Start", "Stop")
|
||||
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
|
||||
t.Errorf("start: %+v", a)
|
||||
}
|
||||
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
|
||||
t.Errorf("stop: %+v", a)
|
||||
}
|
||||
|
||||
// Many modules are counted, not listed in the headline.
|
||||
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
||||
got = watchWaits(f)
|
||||
if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") {
|
||||
t.Errorf("%+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
||||
// account's own service manager (exit-code)". The operator restarts it from the notification.
|
||||
func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
|
||||
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
||||
plainExample(t, o, "openrazer not working on g14",
|
||||
"Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
|
||||
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
|
||||
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
|
||||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
|
||||
t.Errorf("restart: %+v", a)
|
||||
}
|
||||
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
|
||||
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
|
||||
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
|
||||
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
|
||||
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
|
||||
}
|
||||
}
|
||||
|
||||
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
|
||||
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …"
|
||||
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
|
||||
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
|
||||
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
|
||||
{Unit: "greenclip.service", Scope: "user", Result: "exit-code"}}})
|
||||
if !raise {
|
||||
t.Fatal("not raised")
|
||||
}
|
||||
plainExample(t, o, "3 failed services on shanks",
|
||||
"Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+
|
||||
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+
|
||||
"mesh does not repair them.", "Silence for a week")
|
||||
}
|
||||
|
||||
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through
|
||||
// the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup.
|
||||
func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
for i := 0; i < 35; i++ {
|
||||
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), "push")
|
||||
a.Verb, a.By = "push", "g14.node-tools, through the mesh-controller seat"
|
||||
f.handActs = append(f.handActs, a)
|
||||
}
|
||||
got := watchHandActs(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
plainExample(t, got[0], "Push keeps being fixed by hand",
|
||||
"Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+
|
||||
"wanted for it. Nothing is broken now.")
|
||||
}
|
||||
|
||||
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
|
||||
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
|
||||
func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
|
||||
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
||||
plainExample(t, got[0], "Delivery of hq held for 36 hours",
|
||||
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
|
||||
"Release", "Stop")
|
||||
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
|
||||
t.Errorf("release: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
||||
// shape it is raised with.
|
||||
func TestEveryWordingIsPlain(t *testing.T) {
|
||||
subjects := []conditions.Observation{
|
||||
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
|
||||
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
|
||||
{Scope: conditions.ScopeProvider, ID: "postgres.anchor.app_db", Machine: "anchor"},
|
||||
{Scope: conditions.ScopePlan, ID: "plan-1791454185265004861"},
|
||||
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
|
||||
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
|
||||
{Scope: conditions.ScopeBus, ID: "mesh"},
|
||||
{Scope: conditions.ScopeSeat, ID: "node-backup.ace", Machine: "ace"},
|
||||
{Scope: conditions.ScopeMesh, ID: "hand-acts.disk-load"},
|
||||
{Scope: conditions.ScopeBuild, ID: "openrazer.g14", Machine: "g14"},
|
||||
}
|
||||
for kind, fn := range plainWordings {
|
||||
if !conditions.Worded(kind) {
|
||||
t.Errorf("%s is not registered", kind)
|
||||
}
|
||||
for _, s := range subjects {
|
||||
s.Kind = kind
|
||||
w := fn(s)
|
||||
w.Explanation = conditions.Verdict(w.Needs, w.Explanation)
|
||||
if why, ok := conditions.PlainWords(w, s.Machine); !ok {
|
||||
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -334,6 +334,15 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
done := map[string]bool{except: true}
|
||||
// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
|
||||
// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
|
||||
// left without a verdict is one more move every other walk would wait for.
|
||||
if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 {
|
||||
notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", "))
|
||||
for _, m := range kept {
|
||||
done[m] = true
|
||||
}
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
|
||||
continue
|
||||
@@ -356,6 +365,34 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
p.Note = strings.Join(notes, "; ")
|
||||
}
|
||||
|
||||
// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the
|
||||
// send failed (ADR 0254), except the plan's own module; answers the modules kept.
|
||||
func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string {
|
||||
var kept []string
|
||||
seen := map[string]bool{}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) {
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why)
|
||||
if w := g.Waits[c.Module]; w != "" {
|
||||
why += "; and it waits for a person: " + w
|
||||
}
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
|
||||
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module,
|
||||
c.Node, err)
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(kept, c.Module) {
|
||||
kept = append(kept, c.Module)
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// sentTheBuild is every machine running a module that was last sent this build of it.
|
||||
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
|
||||
running, err := open.inventory.Running(ctx, module)
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the
|
||||
// older commit: the statements are read from the node-engine's JSON, as the controller hears them.
|
||||
//
|
||||
// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send —
|
||||
// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged
|
||||
// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the
|
||||
// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service
|
||||
// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its
|
||||
// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and
|
||||
// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there.
|
||||
//
|
||||
// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along,
|
||||
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
||||
// for the operator, that a new login is needed on the laptop.
|
||||
func TestReplay318(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
build := func(module, repository, commit string, asked time.Time) {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
||||
Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
type move struct{ module, repository, from, to string }
|
||||
moves := []move{
|
||||
{"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"},
|
||||
{"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"},
|
||||
{"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"},
|
||||
}
|
||||
machines := []string{"laptop", "workstation"}
|
||||
if _, err := inv.AddNode(ctx, "workstation"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, mv := range moves {
|
||||
build(mv.module, mv.repository, mv.from, old)
|
||||
for _, n := range machines {
|
||||
if _, err := inv.Assign(ctx, n, mv.module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range machines {
|
||||
sent := map[string]string{}
|
||||
for _, mv := range moves {
|
||||
sent[mv.module] = mv.from
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, mv := range moves {
|
||||
build(mv.module, mv.repository, mv.to, recent)
|
||||
}
|
||||
|
||||
// What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the
|
||||
// daemon's unit failed in that account's own manager; docker's container is healthy.
|
||||
statement := func(at time.Time) link.Health {
|
||||
raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [
|
||||
{"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service",
|
||||
"state": "healthy", "since": %q},
|
||||
{"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator",
|
||||
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
|
||||
"reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"},
|
||||
{"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service",
|
||||
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
|
||||
"reason": "failed in the account's own service manager (exit-code)"}]}`,
|
||||
link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano),
|
||||
at.Format(time.RFC3339Nano))
|
||||
var h link.Health
|
||||
if err := json.Unmarshal([]byte(raw), &h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
||||
})
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true, "workstation": true}, nil
|
||||
}
|
||||
var sends [][]string
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
carried := map[string]string{}
|
||||
for _, m := range modules {
|
||||
carried[m] = current[m].Commit
|
||||
}
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
// Every judging hears each machine's newest statement first, as the controller does between judgings:
|
||||
// the account's wait and the failed unit are raised as openrazer's condition, after the send.
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
now := time.Now()
|
||||
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{},
|
||||
judged: true}
|
||||
for _, m := range machines {
|
||||
if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.served[m] = served{runtime: true, tools: map[string]bool{}}
|
||||
}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
if f.health, err = open.inventory.Healths(ctx); err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.open, f.openErr = keeper.Open(ctx)
|
||||
return f, nil
|
||||
}
|
||||
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 0
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, open)
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
var p inventory.Plan
|
||||
plans, err := inv.RecentPlans(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.Release != nil {
|
||||
p = q
|
||||
break
|
||||
}
|
||||
}
|
||||
if p.Release == nil {
|
||||
t.Fatal("no walk of the builds waiting for a gate was opened")
|
||||
}
|
||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) {
|
||||
t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State,
|
||||
p.Release.Done, p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) {
|
||||
t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if current["openrazer"].Commit != "c78b5fc9" {
|
||||
t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit)
|
||||
}
|
||||
for _, mv := range moves {
|
||||
v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to)
|
||||
if err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err)
|
||||
}
|
||||
if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") {
|
||||
t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why)
|
||||
}
|
||||
}
|
||||
// What the operator reads: one condition per machine for openrazer, saying a new login is needed there,
|
||||
// a warning for a person, and no fault of the build.
|
||||
list, err := keeper.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said []string
|
||||
for _, c := range list {
|
||||
said = append(said, c.Key+": "+c.Summary)
|
||||
if c.Subject.ID == "openrazer.laptop" {
|
||||
if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent ||
|
||||
c.Resolver != conditions.ResolverOperator {
|
||||
t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") {
|
||||
t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,20 @@ func TestMain(m *testing.M) {
|
||||
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
|
||||
unsaid.note(o, field, r)
|
||||
}
|
||||
conditions.Unworded = func(o conditions.Observation, why string) {
|
||||
if o.Source != "test" { // a kind a test makes up to exercise the keeper
|
||||
unworded.add(o, why)
|
||||
}
|
||||
}
|
||||
code := m.Run()
|
||||
if said := unworded.all(); len(said) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+
|
||||
"words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+
|
||||
"reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n "))
|
||||
if code == 0 {
|
||||
code = 1
|
||||
}
|
||||
}
|
||||
if said := unsaid.all(); len(said) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+
|
||||
"(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n",
|
||||
@@ -74,6 +87,34 @@ func (u *unsayable) all() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// unworded collects, across the suite, every finding said in borrowed words (ADR 0253).
|
||||
var unworded wordless
|
||||
|
||||
type wordless struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]bool
|
||||
}
|
||||
|
||||
func (u *wordless) add(o conditions.Observation, why string) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
if u.seen == nil {
|
||||
u.seen = map[string]bool{}
|
||||
}
|
||||
u.seen[fmt.Sprintf("kind %s (scope %s, source %s): %s", o.Kind, o.Scope, o.Source, why)] = true
|
||||
}
|
||||
|
||||
func (u *wordless) all() []string {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
var out []string
|
||||
for s := range u.seen {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// linted is a producer's findings, held to the content rule as a keeper would hold them: for a test
|
||||
// that reads a producer's findings without raising them.
|
||||
func linted(obs []conditions.Observation) []conditions.Observation {
|
||||
@@ -85,6 +126,18 @@ func linted(obs []conditions.Observation) []conditions.Observation {
|
||||
if r, ok := outward.Check(o.Summary, machines...); !ok {
|
||||
unsaid.note(o, "summary", r)
|
||||
}
|
||||
switch {
|
||||
case o.Headline != "":
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved}
|
||||
if w.Resolved == "" {
|
||||
w.Resolved = "Resolved"
|
||||
}
|
||||
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
||||
unworded.add(o, "its source's words are not plain: "+why)
|
||||
}
|
||||
case o.Kind != "" && !conditions.Worded(o.Kind):
|
||||
unworded.add(o, "the kind "+o.Kind+" has no plain words")
|
||||
}
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
@@ -322,6 +322,10 @@ func watchPlans(f *signalFacts) []conditions.Observation {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled",
|
||||
Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway",
|
||||
Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+
|
||||
"usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))),
|
||||
Resolved: "Delivery of " + repoName(p.repository) + " moves again",
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s",
|
||||
p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"),
|
||||
@@ -353,7 +357,12 @@ func watchWaits(f *signalFacts) []conditions.Observation {
|
||||
Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+
|
||||
"delivery that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository,
|
||||
short(w.commit), ago(in), w.awaits, short(w.commit), w.id),
|
||||
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits)})
|
||||
Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits),
|
||||
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
|
||||
Explanation: walkWaitingWords(w, in, severity),
|
||||
Needs: waitingNeeds(severity),
|
||||
Actions: waitingActions(w, severity),
|
||||
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -681,7 +690,12 @@ func watchHandActs(f *signalFacts) []conditions.Observation {
|
||||
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause,
|
||||
repeated[cause], int(handActsWithin.Hours()/24), newest.By, wanted),
|
||||
Said: strings.Join(acts, "; ")})
|
||||
Said: strings.Join(acts, "; "),
|
||||
Headline: conditions.Capital(causeWords(cause)) + " keeps being fixed by hand",
|
||||
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is "+
|
||||
"wanted for it. Nothing is broken now.", causeWords(cause), repeated[cause],
|
||||
int(handActsWithin.Hours()/24)),
|
||||
Resolved: "Resolved: no more hand repairs of " + causeWords(cause)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -150,6 +150,8 @@ type planFacts struct {
|
||||
type waitFacts struct {
|
||||
id, repository, commit, awaits string
|
||||
since time.Time
|
||||
// modules are the modules the walk moves, as a person names the delivery.
|
||||
modules []string
|
||||
}
|
||||
|
||||
type loopFacts struct {
|
||||
@@ -451,7 +453,7 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
|
||||
// S16's, whatever mesh-delivery says or does not say.
|
||||
if p.Waiting() {
|
||||
waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit,
|
||||
awaits: p.Delivery.Awaits, since: p.Created})
|
||||
awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p)})
|
||||
continue
|
||||
}
|
||||
_, paused := pausedWaiting(p, pause, now)
|
||||
|
||||
@@ -177,11 +177,19 @@ func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) {
|
||||
// wrong on every node whose assignment differs, and wrong for a second reason on a node given the
|
||||
// port (ADR 0100). Composed through the whole path, because what proves the placeholder resolves
|
||||
// in an `env` at all is a declaration, not a manifest.
|
||||
// forgeBuilt is every artifact the forge's manifest builds: its code, and since hq ADR 0251 the npm
|
||||
// registry it serves beside it. A manifest resolves only against all of what it asked to be built.
|
||||
func forgeBuilt() []Built {
|
||||
var out []Built
|
||||
for _, name := range []string{"code", "npm-registry"} {
|
||||
out = append(out, Built{Name: name, Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "gitea/" + name + "/blobs/" + bundleDigest, Digest: bundleDigest})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
|
||||
forge, err := catalogueManifest(t, "gitea").Resolve([]Built{{
|
||||
Name: "code", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "gitea/code/blobs/" + bundleDigest, Digest: bundleDigest,
|
||||
}})
|
||||
forge, err := catalogueManifest(t, "gitea").Resolve(forgeBuilt())
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
|
||||
}
|
||||
@@ -234,10 +242,7 @@ func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
|
||||
func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
|
||||
t.Helper()
|
||||
forge := catalogueManifest(t, "gitea")
|
||||
resolved, err := forge.Resolve([]Built{{
|
||||
Name: "code", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "gitea/code/blobs/" + bundleDigest, Digest: bundleDigest,
|
||||
}})
|
||||
resolved, err := forge.Resolve(forgeBuilt())
|
||||
if err != nil {
|
||||
t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
|
||||
}
|
||||
|
||||
@@ -125,8 +125,19 @@ type Condition struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
// Summary is one line in the mesh's words.
|
||||
// Summary is one line in the mesh's words, for whoever looks closer: it may name plans, commits and
|
||||
// the verbs that act.
|
||||
Summary string `json:"summary"`
|
||||
// Headline, Explanation and Resolved are what the operator reads, in plain words (novox/hq ADR 0253,
|
||||
// plain.go): a few words naming the thing and what is wrong; one or two sentences on what happened,
|
||||
// what it means and whether to act; the one line said when it clears.
|
||||
Headline string `json:"headline"`
|
||||
Explanation string `json:"explanation"`
|
||||
Resolved string `json:"resolved"`
|
||||
// Needs is what the operator does about it, one sentence; empty when nothing (the explanation then
|
||||
// opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go).
|
||||
Needs string `json:"needs"`
|
||||
Actions []Action `json:"actions"`
|
||||
// Evidence is the newest observations, at most KeptEvidence, newest first.
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||
@@ -175,6 +186,13 @@ type Observation struct {
|
||||
Also []string
|
||||
Severity Severity
|
||||
Summary string
|
||||
// Headline, Explanation and Resolved are the plain words the operator reads (plain.go). Left empty,
|
||||
// the wording registered for Kind says them.
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
Needs string
|
||||
Actions []Action
|
||||
// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
|
||||
// here, never in Summary**: an address, a socket's error, a path or a name with its domain is
|
||||
// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
package conditions
|
||||
|
||||
// Plain words (novox/hq ADR 0253): **a condition carries what the operator reads, in plain words, beside
|
||||
// what an agent reads.** The summary is one line in the mesh's words for whoever looks closer — it names
|
||||
// plans, commits and the verbs that act, and stays so. The operator reads a notification between other
|
||||
// work: a popup that said "the walk of novox/mesh-catalog a6385479 has waited 56m0s for mesh-delivery's
|
||||
// word to start: `mesh-delivery.show` …" was a wall of identifiers nobody could act on. So every condition
|
||||
// also carries:
|
||||
//
|
||||
// - a **headline**: a few plain words naming the thing and what is wrong ("openrazer not running on
|
||||
// g14"), the title of every message about it;
|
||||
// - an **explanation**: one or two plain sentences — what happened, what it means for the operator,
|
||||
// and whether they need to do anything;
|
||||
// - a **resolved line**: the one short line said when it clears ("openrazer runs again on g14");
|
||||
// - a **verdict**, which every explanation opens with: "Nothing for you to do." or "Needs you:" and one
|
||||
// concrete thing the operator can do themselves (needs). Never "have an agent …": the operator is not
|
||||
// asked to open a session to understand or answer a notification;
|
||||
// - **actions**: what the operator may answer from the notification itself (Release, Stop, Restart,
|
||||
// Silence), each the seat verb the operator channel calls when it is chosen, naming the operator.
|
||||
//
|
||||
// They are made here, where the condition is made, so every channel gets them — the desktop today,
|
||||
// others later — and none has to guess what a key means. A producer may say them itself; otherwise the
|
||||
// wording registered for its kind says them; otherwise a plain sentence made from its scope does, and a
|
||||
// test suite is told (Unworded) so the kind gets words of its own. Each is held to the plain rule (Plain):
|
||||
// no identifiers, hashes, keys, verb syntax, markup or clock times — a message names no time of its own,
|
||||
// since the channel says when, in the operator's time.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/outward"
|
||||
)
|
||||
|
||||
// Words are what the operator reads of a condition. Explanation is said after the verdict, which
|
||||
// plainly writes from Needs: empty is "Nothing for you to do.", else "Needs you: " and Needs.
|
||||
type Words struct {
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
Needs string
|
||||
Actions []Action
|
||||
}
|
||||
|
||||
// Action is one answer the operator may give from a notification: a label, and the seat verb the
|
||||
// operator channel calls with these arguments (and a why naming the operator and the label) when it is
|
||||
// chosen. Machine is set for a seat every machine holds. An argument "why" given empty is the operator
|
||||
// channel's to fill: it names the operator, the channel and the label chosen.
|
||||
type Action struct {
|
||||
Label string `json:"label"`
|
||||
Verb string `json:"verb"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Arguments map[string]string `json:"arguments,omitempty"`
|
||||
}
|
||||
|
||||
// The two verdicts an explanation opens with.
|
||||
const (
|
||||
NothingToDo = "Nothing for you to do."
|
||||
NeedsYou = "Needs you:"
|
||||
)
|
||||
|
||||
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
|
||||
// answer to a condition the operator decided to live with.
|
||||
func SilenceAction(key string) Action {
|
||||
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
|
||||
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
|
||||
}
|
||||
|
||||
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
|
||||
const (
|
||||
HeadlineMax = 60
|
||||
ExplanationMax = 420
|
||||
NeedsMax = 120
|
||||
)
|
||||
|
||||
var (
|
||||
wordingsMu sync.RWMutex
|
||||
wordings = map[string]func(Observation) Words{}
|
||||
)
|
||||
|
||||
// Wording registers the plain words of one condition kind.
|
||||
func Wording(kind string, words func(Observation) Words) {
|
||||
wordingsMu.Lock()
|
||||
defer wordingsMu.Unlock()
|
||||
wordings[kind] = words
|
||||
}
|
||||
|
||||
// Worded says whether a kind has words of its own.
|
||||
func Worded(kind string) bool {
|
||||
wordingsMu.RLock()
|
||||
defer wordingsMu.RUnlock()
|
||||
_, ok := wordings[kind]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Unworded is told of every observation said in borrowed words: its kind has none registered, or the
|
||||
// words it was given break the plain rule. The keeper says it plainly anyway; a test suite sets this to
|
||||
// fail the producer.
|
||||
var Unworded func(o Observation, why string)
|
||||
|
||||
// The plain rule's shapes.
|
||||
var (
|
||||
hexID = regexp.MustCompile(`\b[0-9a-f]{7,40}\b`)
|
||||
numberedID = regexp.MustCompile(`\b[a-z]+-[0-9]{6,}\b`)
|
||||
dotted = regexp.MustCompile(`\b[a-z][a-z0-9_-]*\.[a-z][a-z0-9_-]*\b`)
|
||||
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
|
||||
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
|
||||
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
|
||||
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
|
||||
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
|
||||
)
|
||||
|
||||
// Plain says whether text is plain words the operator reads at a glance, and if not, what is not: an
|
||||
// identifier (a hash, a numbered id, a dotted name such as a key or a verb), a Go duration, a clock time
|
||||
// or date (the channel says when, in the operator's time), a command's flag, markup, a line break, or
|
||||
// anything the operator's channel withholds (ADR 0234 §6). machines are names that may appear.
|
||||
func Plain(text string, machines ...string) (string, bool) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return "nothing is said", false
|
||||
}
|
||||
if strings.ContainsAny(text, "\n\r\t") {
|
||||
return "a line break", false
|
||||
}
|
||||
if m := markup.FindString(text); m != "" {
|
||||
return "markup (" + m + ")", false
|
||||
}
|
||||
for _, shape := range []struct {
|
||||
re *regexp.Regexp
|
||||
what string
|
||||
}{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"},
|
||||
{clockTime, "a clock time or date"}, {flag, "a command's flag"}} {
|
||||
if m := shape.re.FindString(text); m != "" {
|
||||
if shape.re == hexID && !strings.ContainsAny(m, "0123456789") {
|
||||
continue // a word of letters a to f only
|
||||
}
|
||||
return shape.what + " (" + strings.TrimSpace(m) + ")", false
|
||||
}
|
||||
}
|
||||
if r, ok := outward.Check(text, machines...); !ok {
|
||||
return r.String(), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// PlainWords checks all three, with their bounds.
|
||||
func PlainWords(w Words, machines ...string) (string, bool) {
|
||||
if why, ok := Plain(w.Headline, machines...); !ok {
|
||||
return "headline: " + why, false
|
||||
}
|
||||
if len(w.Headline) > HeadlineMax {
|
||||
return fmt.Sprintf("headline: longer than %d characters", HeadlineMax), false
|
||||
}
|
||||
if why, ok := Plain(w.Explanation, machines...); !ok {
|
||||
return "explanation: " + why, false
|
||||
}
|
||||
if len(w.Explanation) > ExplanationMax {
|
||||
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
|
||||
}
|
||||
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
|
||||
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
|
||||
return "explanation: sends the operator elsewhere (" + m + ")", false
|
||||
}
|
||||
}
|
||||
if w.Needs != "" {
|
||||
if why, ok := Plain(w.Needs, machines...); !ok {
|
||||
return "needs: " + why, false
|
||||
}
|
||||
if m := agentWord.FindString(w.Needs); m != "" {
|
||||
return "needs: not something the operator does themselves (" + m + ")", false
|
||||
}
|
||||
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
|
||||
return fmt.Sprintf("needs: one sentence of at most %d characters, ending in a full stop", NeedsMax), false
|
||||
}
|
||||
}
|
||||
for _, a := range w.Actions {
|
||||
if a.Label == "" || len(a.Label) > 24 || a.Verb == "" {
|
||||
return fmt.Sprintf("action %q: a short label and a verb", a.Label), false
|
||||
}
|
||||
}
|
||||
if why, ok := Plain(w.Resolved, machines...); !ok {
|
||||
return "resolved: " + why, false
|
||||
}
|
||||
if len(w.Resolved) > HeadlineMax+20 {
|
||||
return fmt.Sprintf("resolved: longer than %d characters", HeadlineMax+20), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// plainly gives an observation its plain words: its own, its kind's, or its scope's.
|
||||
func plainly(o Observation) Observation {
|
||||
machines := append([]string{o.Machine}, o.Also...)
|
||||
given := Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
||||
Actions: o.Actions}
|
||||
var w Words
|
||||
from := ""
|
||||
switch {
|
||||
case given.Headline != "":
|
||||
w, from = given, "its source"
|
||||
default:
|
||||
wordingsMu.RLock()
|
||||
fn := wordings[o.Kind]
|
||||
wordingsMu.RUnlock()
|
||||
if fn != nil {
|
||||
w, from = fn(o), "the wording of "+o.Kind
|
||||
if given.Explanation != "" {
|
||||
w.Explanation = given.Explanation
|
||||
}
|
||||
if given.Resolved != "" {
|
||||
w.Resolved = given.Resolved
|
||||
}
|
||||
if given.Needs != "" {
|
||||
w.Needs = given.Needs
|
||||
}
|
||||
if len(given.Actions) > 0 {
|
||||
w.Actions = given.Actions
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.Headline != "" && w.Resolved == "" {
|
||||
w.Resolved = "Resolved: " + lowerFirst(w.Headline)
|
||||
}
|
||||
if from == "" {
|
||||
if Unworded != nil {
|
||||
Unworded(o, "the kind "+o.Kind+" has no plain words")
|
||||
}
|
||||
w = scopeWords(o)
|
||||
} else if why, ok := PlainWords(w, machines...); !ok {
|
||||
if Unworded != nil {
|
||||
Unworded(o, from+" is not plain: "+why)
|
||||
}
|
||||
w = scopeWords(o)
|
||||
}
|
||||
o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
|
||||
o.Explanation = Verdict(w.Needs, w.Explanation)
|
||||
return o
|
||||
}
|
||||
|
||||
// Verdict is an explanation opened by its verdict.
|
||||
func Verdict(needs, explanation string) string {
|
||||
if needs == "" {
|
||||
return strings.TrimSpace(NothingToDo + " " + explanation)
|
||||
}
|
||||
return strings.TrimSpace(NeedsYou + " " + needs + " " + explanation)
|
||||
}
|
||||
|
||||
// Escalated is what a condition needs once a healer gave up on it, where its words needed nothing.
|
||||
const Escalated = "the mesh tried to repair this and could not; read the details to decide what to do."
|
||||
|
||||
// escalatedWords turns "Nothing for you to do." into "Needs you:" once a healer gave up.
|
||||
func escalatedWords(c *Condition) {
|
||||
if c.Escalated() && c.Needs == "" {
|
||||
c.Needs = Escalated
|
||||
c.Explanation = Verdict(c.Needs, strings.TrimSpace(strings.TrimPrefix(c.Explanation, NothingToDo)))
|
||||
}
|
||||
}
|
||||
|
||||
// scopeWords is what is said of a kind with no words of its own: what kind of thing, and which one
|
||||
// where its name is a name (a machine, a module), never an id.
|
||||
func scopeWords(o Observation) Words {
|
||||
thing := ThingWords(o)
|
||||
what := strings.ReplaceAll(o.Kind, "-", " ")
|
||||
w := Words{
|
||||
Headline: Capital(thing) + " needs a look",
|
||||
Explanation: fmt.Sprintf("The mesh noticed a problem it calls %q with %s.", what, thing),
|
||||
Resolved: "Resolved: " + thing + " is fine again",
|
||||
Needs: ResolverNeeds(o),
|
||||
}
|
||||
if _, ok := PlainWords(w, append([]string{o.Machine}, o.Also...)...); !ok {
|
||||
w = Words{Headline: "Something in the mesh needs a look",
|
||||
Explanation: "The mesh noticed a problem it has no plain words for yet.",
|
||||
Resolved: "Resolved: the mesh is fine again", Needs: ResolverNeeds(o)}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// ThingWords is what a condition is about, in plain words: "the machine ace", "openrazer on g14", "a
|
||||
// delivery". An id that is not a name (a plan's, a delivery's, a call's) is never said.
|
||||
func ThingWords(o Observation) string {
|
||||
switch o.Scope {
|
||||
case ScopeMachine:
|
||||
if o.Machine != "" {
|
||||
return o.Machine
|
||||
}
|
||||
return "a machine"
|
||||
case ScopeModule:
|
||||
module := strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
if o.Machine != "" && module != "" && !strings.Contains(module, ".") {
|
||||
return module + " on " + o.Machine
|
||||
}
|
||||
return "a module"
|
||||
case ScopeProvider:
|
||||
if module, _, _ := strings.Cut(o.ID, "."); module != "" && o.Machine != "" {
|
||||
return module + " on " + o.Machine
|
||||
}
|
||||
return "a provider"
|
||||
case ScopePlan:
|
||||
return "a walk"
|
||||
case ScopeDelivery:
|
||||
return "a delivery"
|
||||
case ScopeBuild:
|
||||
return "a build"
|
||||
case ScopeCall:
|
||||
return "a tool call"
|
||||
case ScopeMerge:
|
||||
return "a merge"
|
||||
case ScopeSeat:
|
||||
return "a seat's holder"
|
||||
case ScopeBus:
|
||||
return "the bus"
|
||||
case ScopeCore:
|
||||
return "the controller"
|
||||
case ScopeProbe:
|
||||
return "the self-check"
|
||||
}
|
||||
return "the mesh"
|
||||
}
|
||||
|
||||
// ResolverNeeds is what the operator needs to do, from who resolves it: nothing, unless only a person can.
|
||||
func ResolverNeeds(o Observation) string {
|
||||
if o.Resolver == ResolverOperator {
|
||||
return "the mesh does not repair this by itself; read the details to decide what to do."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Capital is s with its first letter upper case.
|
||||
func Capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
|
||||
func lowerFirst(s string) string {
|
||||
if s == "" || (len(s) > 1 && strings.ToUpper(s[:2]) == s[:2]) {
|
||||
return s
|
||||
}
|
||||
return strings.ToLower(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package conditions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The plain rule** (novox/hq ADR 0253): what the operator reads names no identifier, no code and no time
|
||||
// of its own; the words conditions are made of pass.
|
||||
func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) {
|
||||
refused := map[string]string{
|
||||
"the walk of novox/mesh-catalog a6385479 waits": "a hash",
|
||||
"plan plan-1791454185265004861 is waiting": "a hash",
|
||||
"see `mesh-delivery.show` for why": "markup",
|
||||
"call mesh-delivery.show for why": "a dotted name",
|
||||
"key plan.x.waiting": "a dotted name",
|
||||
"waited 56m0s": "a duration in code",
|
||||
"since 10:40": "a clock time",
|
||||
"since 2026-10-08": "a clock time",
|
||||
"raised at noon UTC": "a clock time",
|
||||
"start it with plans go --why": "a command's flag",
|
||||
"one line\nand another": "a line break",
|
||||
"the resolver at 10.77.0.1 is silent": "",
|
||||
"": "nothing",
|
||||
"**bold**": "markup",
|
||||
"storage-media.mount failed": "a dotted name",
|
||||
"the store at /var/lib/mesh is full": "",
|
||||
"a commit 0123abc landed": "a hash",
|
||||
}
|
||||
for text, want := range refused {
|
||||
why, ok := Plain(text, "g14")
|
||||
if ok {
|
||||
t.Errorf("%q passed", text)
|
||||
continue
|
||||
}
|
||||
if want != "" && !strings.HasPrefix(why, want) {
|
||||
t.Errorf("%q: refused for %q, want %q", text, why, want)
|
||||
}
|
||||
}
|
||||
for _, text := range []string{
|
||||
"openrazer delivery waiting to start",
|
||||
"The change to openrazer is merged and built, and has waited 56 minutes for mesh-delivery to let it start.",
|
||||
"3 failed services on shanks",
|
||||
"On shanks, mnt-recalbox (a mount), storage-media (a mount) and greenclip failed.",
|
||||
"Name lookups fail on g14",
|
||||
"Retired data of postgres on anchor waits for app_db",
|
||||
"Push keeps being fixed by hand",
|
||||
} {
|
||||
if why, ok := Plain(text, "g14", "shanks", "anchor"); !ok {
|
||||
t.Errorf("%q refused: %s", text, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A condition carries its plain words**: a producer's own; else its kind's; else its scope's — and the
|
||||
// event says them beside the summary, for every channel.
|
||||
func TestAConditionCarriesItsPlainWords(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
before := Unworded
|
||||
var unworded []string
|
||||
Unworded = func(o Observation, why string) { unworded = append(unworded, why) }
|
||||
t.Cleanup(func() { Unworded = before })
|
||||
|
||||
Wording("test-silent", func(o Observation) Words {
|
||||
return Words{Headline: o.Machine + " is not answering", Explanation: "The mesh has not heard from it.",
|
||||
Resolved: o.Machine + " answers again"}
|
||||
})
|
||||
c, err := k.Observe(t.Context(), Observation{Scope: ScopeMachine, ID: "ace", Kind: "test-silent", Machine: "ace",
|
||||
Severity: Warning, Source: "S1", Summary: "ace has not been heard from since 12:00 UTC (bound 3m0s)"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Headline != "ace is not answering" || c.Resolved != "ace answers again" || c.Explanation == "" {
|
||||
t.Fatalf("the kind's words were not kept: %+v", c)
|
||||
}
|
||||
if c.Summary != "ace has not been heard from since 12:00 UTC (bound 3m0s)" {
|
||||
t.Errorf("the summary was changed: %q", c.Summary)
|
||||
}
|
||||
said := settled(t, told, 1)
|
||||
body, _ := json.Marshal(said[0])
|
||||
for _, field := range []string{`"headline":"ace is not answering"`, `"explanation":"Nothing for you to do. The mesh has not heard from it."`,
|
||||
`"resolved":"ace answers again"`} {
|
||||
if !strings.Contains(string(body), field) {
|
||||
t.Errorf("the event does not carry %s: %s", field, body)
|
||||
}
|
||||
}
|
||||
|
||||
// A producer's own words win; a resolved line it leaves out is made from its headline.
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Kind: "test-silent",
|
||||
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy",
|
||||
Headline: "openrazer not working on g14", Explanation: "Its service stopped with an error."})
|
||||
if c.Headline != "openrazer not working on g14" || c.Resolved != "Resolved: openrazer not working on g14" {
|
||||
t.Errorf("the producer's words: %+v", c)
|
||||
}
|
||||
if len(unworded) != 0 {
|
||||
t.Errorf("words that were plain were reported: %v", unworded)
|
||||
}
|
||||
|
||||
// Words that are not plain, and a kind with none, are said from the scope — and reported.
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-unworded",
|
||||
Machine: "g14", Severity: Warning, Source: "health", Summary: "openrazer on g14 is not healthy"})
|
||||
if c.Headline != "Openrazer on g14 needs a look" || !strings.HasPrefix(c.Explanation, NothingToDo) {
|
||||
t.Errorf("the scope's words: %+v", c)
|
||||
}
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopePlan, ID: "plan-1791454185265004861", Kind: "test-silent",
|
||||
Severity: Warning, Source: "S16", Summary: "the walk waits", Headline: "plan-1791454185265004861 waits"})
|
||||
if strings.Contains(c.Headline, "plan-") || c.Headline != "A walk needs a look" {
|
||||
t.Errorf("an id reached the headline: %+v", c)
|
||||
}
|
||||
if len(unworded) != 2 || !strings.Contains(unworded[0], "no plain words") || !strings.Contains(unworded[1], "not plain") {
|
||||
t.Errorf("what was said in borrowed words was not reported: %v", unworded)
|
||||
}
|
||||
}
|
||||
|
||||
// **An explanation opens with its verdict, and never sends the operator to an agent.**
|
||||
func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
|
||||
if got := Verdict("", "It may be asleep."); got != "Nothing for you to do. It may be asleep." {
|
||||
t.Errorf("%q", got)
|
||||
}
|
||||
if got := Verdict("release it, or stop it.", "It is held."); got != "Needs you: release it, or stop it. It is held." {
|
||||
t.Errorf("%q", got)
|
||||
}
|
||||
for _, w := range []Words{
|
||||
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
|
||||
} {
|
||||
if _, ok := PlainWords(w); ok {
|
||||
t.Errorf("passed: %+v", w)
|
||||
}
|
||||
}
|
||||
if why, ok := PlainWords(Words{Headline: "push keeps being fixed by hand", Explanation: "A person repaired " +
|
||||
"push by hand 35 times.", Resolved: "Resolved", Needs: "release it, or stop it.",
|
||||
Actions: []Action{{Label: "Release", Verb: "mesh-delivery.release"}}}); !ok {
|
||||
t.Errorf("refused: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// **A healer that gave up says so in the explanation**: the words of the kind said "nothing to do"; once the
|
||||
// budget is spent, the operator is needed.
|
||||
func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
|
||||
k, _, _, _ := keeper(t)
|
||||
ctx := t.Context()
|
||||
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
|
||||
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _, err := k.Escalate(ctx, o.Key(), Attempt{What: "budget spent", Outcome: "escalated", By: "healer H1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "Needs you: " + Escalated + " It may be asleep."; c.Explanation != want || c.Needs != Escalated {
|
||||
t.Fatalf("escalated: %q, want %q", c.Explanation, want)
|
||||
}
|
||||
c, _ = k.Observe(ctx, o)
|
||||
if strings.Count(c.Explanation, Escalated) != 1 || strings.Contains(c.Explanation, NothingToDo) {
|
||||
t.Errorf("seen again after escalation: %q", c.Explanation)
|
||||
}
|
||||
}
|
||||
@@ -175,6 +175,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
return Condition{}, err
|
||||
}
|
||||
o = k.sayable(o)
|
||||
o = plainly(o)
|
||||
key := o.Key()
|
||||
for i := 0; i < tries; i++ {
|
||||
now := k.now().UTC()
|
||||
@@ -188,7 +189,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
}
|
||||
if !found {
|
||||
c := Condition{Key: key, Kind: o.Kind, Subject: Subject{Scope: o.Scope, ID: o.ID, Machine: o.Machine, Also: o.Also},
|
||||
Severity: o.Severity, Summary: o.Summary, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Severity: o.Severity, Summary: o.Summary, Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions, Evidence: []Evidence{{At: now, Said: said}},
|
||||
Source: o.Source, Raised: now, LastObserved: now, Observations: 1, Count: 1,
|
||||
Resolver: orSelf(o.Resolver)}
|
||||
change := ChangeRaised
|
||||
@@ -238,6 +240,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
// The kind as the source says it now: a source that gave the same key a kind of its own since
|
||||
// (a probe's finding split out for a healer) is read by that kind from its next observation.
|
||||
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
|
||||
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
|
||||
escalatedWords(&c)
|
||||
if o.Machine != "" {
|
||||
c.Subject.Machine = o.Machine
|
||||
}
|
||||
@@ -335,6 +339,7 @@ func (k *Keeper) Escalate(ctx context.Context, key string, a Attempt) (Condition
|
||||
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
|
||||
c.Resolver = ResolverOperator
|
||||
}
|
||||
escalatedWords(c)
|
||||
return changes
|
||||
})
|
||||
}
|
||||
|
||||
@@ -28,6 +28,9 @@ type ResourceHealth struct {
|
||||
// (ADR 0240 Phase B, to-be 48 §6).
|
||||
Check string `json:"check,omitempty"`
|
||||
Needs string `json:"needs,omitempty"`
|
||||
// Account is the account whose own service manager runs it, or the account a resource of kind account
|
||||
// is (novox/hq ADR 0254).
|
||||
Account string `json:"account,omitempty"`
|
||||
}
|
||||
|
||||
// NodeHealth is a machine's newest statement, as kept.
|
||||
|
||||
@@ -143,6 +143,15 @@ type PlanGate struct {
|
||||
Carried []CarriedMove `json:"carried,omitempty"`
|
||||
// Failing names the modules the last judging found wanting.
|
||||
Failing []string `json:"failing,omitempty"`
|
||||
// Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged,
|
||||
// or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's.
|
||||
Healthy map[string]int `json:"healthy,omitempty"`
|
||||
// Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never
|
||||
// a reason to fail.
|
||||
Waits map[string]string `json:"waits,omitempty"`
|
||||
// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
|
||||
// pass, and is not put back.
|
||||
Passing []string `json:"passing,omitempty"`
|
||||
}
|
||||
|
||||
// CarriedMove is one module's build moving on a machine with a gated send.
|
||||
|
||||
@@ -441,6 +441,15 @@ type Health struct {
|
||||
// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
|
||||
const KindUnit = "unit"
|
||||
|
||||
// KindAccount is an account a module puts in a group (novox/hq ADR 0252): healthy when the account's running
|
||||
// session has every group the module declares, or nobody is logged in; unhealthy otherwise, its reason
|
||||
// starting ReasonRelogin when only a new login is missing.
|
||||
const KindAccount = "account"
|
||||
|
||||
// ReasonRelogin starts the reason of an account whose running session began before it was put in a group
|
||||
// (ADR 0252): the build did what it should, and a person has one step left (novox/hq ADR 0254).
|
||||
const ReasonRelogin = "relogin needed"
|
||||
|
||||
// The states of a machine's service managers (issue 315).
|
||||
const (
|
||||
UnitsRunning = "running"
|
||||
@@ -529,6 +538,10 @@ type ResourceHealth struct {
|
||||
// alone; Needs is the provision the check exercises (to-be 48 §6).
|
||||
Check string `json:"check,omitempty"`
|
||||
Needs string `json:"needs,omitempty"`
|
||||
// Account is the account whose own service manager runs it, for a unit or service in an account's
|
||||
// manager, and the account itself for a resource of kind KindAccount (novox/hq ADR 0254). Empty from an
|
||||
// engine older than that, and for anything the machine's own manager or runtime runs.
|
||||
Account string `json:"account,omitempty"`
|
||||
}
|
||||
|
||||
// HealthSaid is the health event's body: the machine and its statement. The machine is read from the
|
||||
|
||||
Reference in New Issue
Block a user