package main import ( "context" "fmt" "sort" "strings" "sync/atomic" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5, // Phase A). // // **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A // machine states, in every report and as an event between reports, the state of every long-running // resource it runs for a module. The controller keeps the newest statement per machine (node_health), and // raises `module...unhealthy` when two statements in a row say a resource of the module // is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be // wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads // the stated health: a judging passes a module only when every long-running resource of it on that // machine is stated healthy, so a resource still starting is not yet a pass. // // **An engine older than the judging states nothing**, and its machine's health is not known: never // healthy, never a reason to raise anything, and the gate judges it as it did before. // The condition a module's health raises. const ( kindModuleUnhealthy = "module-unhealthy" // sourceHealth is what raised it: the machine's own statement. sourceHealth = "health" // moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4). moduleUnhealthyUrgentAfter = 4 * time.Hour // moduleUnhealthyAfter is how many statements in a row raise it. moduleUnhealthyAfter = 2 ) // healthRefused counts the statements refused as older than the one kept, for the log and a test. var healthRefused atomic.Int64 // moduleHealth keeps what the machines state, for the link (link.Healths). type moduleHealth struct { inv *inventory.Inventory keeper func() *conditions.Keeper } func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error { return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now()) } // stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An // older statement than the one kept is refused, by when the engine looked. func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health, now time.Time) error { if h.Contract == 0 { return nil } prev, had, err := inv.HealthOf(ctx, node) if err != nil { return err } if had && h.At.Before(prev.SaidAt) { healthRefused.Add(1) return nil } unhealthy := map[string][]inventory.ResourceHealth{} resources := make([]inventory.ResourceHealth, 0, len(h.Resources)) for _, r := range h.Resources { kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts, Check: r.Check, Needs: r.Needs, Account: r.Account} resources = append(resources, kept) if r.State == link.StateUnhealthy && r.Module != "" { unhealthy[r.Module] = append(unhealthy[r.Module], kept) } } streaks := map[string]int{} for module := range unhealthy { streaks[module] = prev.Streaks[module] + 1 } var network *inventory.NetworkHealth if h.Network != nil { network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}} for _, p := range h.Network.Parts { network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason, Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak}) } } stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At, HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)}) if err != nil || !stored { if err == nil { healthRefused.Add(1) } return err } if k == nil { return nil } err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) // And the machine's own failed units (issue 315): what no module places, one finding for the machine. if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil { if err == nil { err = uerr } else { err = fmt.Errorf("%w; %v", err, uerr) } } // And every machine's network, from every machine's newest statement (ADR 0241): a statement about one // machine can hold another's finding, or release it. if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil { if err == nil { return nerr } return fmt.Errorf("%w; %v", err, nerr) } return err } // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a // resource of it is unhealthy — or on the first while it is already open — and clears every one this // statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48 // §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone // waits on it. func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error { open, err := k.Open(ctx) if err != nil { return err } standing := map[string]conditions.Condition{} for _, c := range open { if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) && c.Subject.Machine == node { standing[c.Key] = c } } var hold *holding if inv != nil { if hold, err = readHolding(ctx, inv, open); err != nil { return err } } var problems []string modules := make([]string, 0, len(unhealthy)) for m := range unhealthy { modules = append(modules, m) } sort.Strings(modules) seen := map[string]bool{} // became is, per module, the kind of condition this statement says of it: what a standing one of the // other kind turned into. became := map[string]string{} heldOn := map[string]string{} providers := map[catalogue.Chosen]bool{} for _, m := range modules { // **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the // machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind, // so the gate never reads it as a fault of the build that happened to be sent beside it. if said, waits := foundWait(m, node, unhealthy[m]); waits { o := usedAsFoundObservation(m, node, said, unhealthy[m]) seen[o.Key()] = true became[m] = kindUsedAsFound if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen { continue } if _, err := k.Observe(ctx, o); err != nil { problems = append(problems, err.Error()) } continue } // **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the // operator, never urgent, cleared on the first statement that no longer says it. if said, waits := personWait(m, node, unhealthy[m]); waits { o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m]) // **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its // consumers are held under it, and its own condition is where they are said. if hold != nil { sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) } seen[o.Key()] = true became[m] = kindReloginNeeded if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen { continue } if _, err := k.Observe(ctx, o); err != nil { problems = append(problems, err.Error()) } continue } o := moduleUnhealthyObservation(m, node, unhealthy[m]) if hold != nil { if p, held := hold.heldUnder(node, m, unhealthy[m]); held { // Held under the provider's condition: nothing of its own, and the provider's says it waits. heldOn[o.Key()] = p.Module + " on " + p.Node providers[p] = true continue } sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) } seen[o.Key()] = true became[m] = kindModuleUnhealthy c, isOpen := standing[o.Key()] if streaks[m] < moduleUnhealthyAfter && !isOpen { continue // unconfirmed: one statement can be wrong; `node show` lists it } if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter { o.Severity = conditions.Urgent } if _, err := k.Observe(ctx, o); err != nil { problems = append(problems, err.Error()) } } for key, c := range standing { if seen[key] { continue } module := strings.TrimSuffix(c.Subject.ID, "."+node) why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module) if c.Kind == kindReloginNeeded { why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module) } if c.Kind == kindUsedAsFound { why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module) } if on, held := heldOn[key]; held { why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on) } // **A condition that became the other kind** is not "working again" (issue 318 review): its clearing // line says what it became. resolved := "" switch { case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded: why = fmt.Sprintf("%s on %s now waits only for a new login", module, node) resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node) case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy: why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node) resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module) } if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil { problems = append(problems, err.Error()) } } // And each provider a consumer here now waits on, when its own condition is open: said again with who // waits on it, so the wait is listed at the provider whichever machine's statement arrived first. for p := range providers { if err := sayWaiters(ctx, k, hold, p, now); err != nil { problems = append(problems, err.Error()) } } if len(problems) > 0 { return fmt.Errorf("%s", strings.Join(problems, "; ")) } return nil } // sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest // statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks. func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error { var raisedAt *conditions.Condition for i, c := range hold.open { if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) { raisedAt = &hold.open[i] } } if raisedAt == nil { return nil } var rs []inventory.ResourceHealth for _, r := range hold.healths[p.Node].Resources { if r.Module == p.Module && r.State == link.StateUnhealthy { rs = append(rs, r) } } if len(rs) == 0 { return nil } o := moduleUnhealthyObservation(p.Module, p.Node, rs) if said, waits := personWait(p.Module, p.Node, rs); waits { o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs) } if o.Key() != raisedAt.Key { return nil // its own statement says it next } sayWaitingOn(&o, hold.waitersOn(p)) _, err := k.Observe(ctx, o) return err } // reloginKey is a module's relogin-needed condition on a machine. func reloginKey(module, node string) string { return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded) } // operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29). func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string { if inv == nil { return "" } n, err := inv.NodeByName(ctx, node) if err != nil { return "" } return n.Account } // reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the // operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its // plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a // person in again, and a restart of the module's service would start it in the same session. operator is // the machine's operator account, when known: the words say "your account" only for it. func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation { o := moduleUnhealthyObservation(module, node, rs) o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said accounts := waitingAccounts(module, rs) yours := operator != "" && len(accounts) > 0 for _, a := range accounts { yours = yours && a == operator } w := reloginWords(module, node, yours) o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil return o } // reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253, // ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says // the account waiting is the operator's own on that machine; otherwise the words do not claim it is. func reloginWords(module, node string, yours bool) words { if yours { return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node), Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node), Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+ "cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module), Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)} } return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node), Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node), Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+ "so %s cannot run until it logs in again. Its update is in place and nothing was undone.", conditions.Capital(module), node, module), Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)} } // waitingAccounts is every account of a module whose resource says it waits for a new login, sorted. func waitingAccounts(module string, rs []inventory.ResourceHealth) []string { seen := map[string]bool{} var out []string for _, r := range rs { if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] { seen[accountOf(r)] = true out = append(out, accountOf(r)) } } sort.Strings(out) return out } // sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone // waits on it, whether it is not working or waits for a new login. func sayWaitingOn(o *conditions.Observation, waiters []string) { if len(waiters) == 0 { return } o.Severity = conditions.Urgent o.Said += "; " + waitingWords(waiters) o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters)) } // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module, // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence. func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { var words, said, plain []string needs := moduleNeeds(node, rs) for _, r := range rs { plain = append(plain, resourcePlainWords(r)) if r.Kind == link.KindUnit { // A failed unit is named by the unit, which is what a person looks for (issue 315); the // resource that places it — a package, a file — is evidence. words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason)) said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"))) continue } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s", r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts, r.Since.UTC().Format("2006-01-02 15:04:05 MST"))) } return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth, Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")), Said: strings.Join(said, "; "), Headline: fmt.Sprintf("%s not working on %s", module, node), Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node, namesWords(plain, 3)), Needs: needs, Resolved: fmt.Sprintf("%s works again on %s", module, node)} } // reasonWords is why a resource is unhealthy, as a person reads it. func reasonWords(r inventory.ResourceHealth) string { // An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's. if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) { return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r)) } switch r.Reason { case "restarting": return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts) case "down": return "is not running" case "": return "is unhealthy" } // What a declared check found says an endpoint, a path or an address: evidence, never the summary the // operator's channel carries (ADR 0234 §6). The summary names the check. if r.Check != "" { return "fails its " + r.Check + " check" } return "is unhealthy: " + r.Reason } func orNotSaid(s string) string { if s == "" { return "no reason said" } return s } // kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq // ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault. const kindReloginNeeded = "relogin-needed" // personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and // that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never // excused: // // - at least one of the module's resources is its account (kind account), unhealthy with a reason that // starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session // running began before; // - every other unhealthy resource of the module runs in the own service manager of one of those very // accounts (Account names it): the manager that began before the group and does not hold it. // // Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in // another account's manager, a resource whose engine does not say whose manager it is in, an account // not in its group or that could not be read — is not a wait, and the module is judged as before. A // resource still starting is not unhealthy and does not make a wait either. Pure. func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) { accounts := waitingAccounts(module, rs) if len(accounts) == 0 { return "", false } waiting := map[string]bool{} for _, a := range accounts { waiting[a] = true } var units []string for _, r := range rs { if r.Module != module || r.State != link.StateUnhealthy { continue } switch { case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]: case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]: units = append(units, r.Target) default: return "", false } } said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+ "login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module, strings.Join(accounts, ", ")) if len(units) > 0 { sort.Strings(units) said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", ")) } return said, true } // accountOf is the account a resource of kind account is: named by the engine, or its target. func accountOf(r inventory.ResourceHealth) string { if r.Account != "" { return r.Account } return r.Target } // moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236 // §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since // the send; not yet otherwise, saying which. A machine that never stated health is judged as before. // // **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module // waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build, // and not something a machine can meet within a bound. func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) { if f.healthErr != nil { return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error()) } h, states := f.health[machine] if !states { return healthGood, "" } if h.HeardAt.Before(since) { return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module) } wait, waits := personWait(module, machine, h.Resources) // **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from // what the controller sent, never from when the machine says the wait began — that time is the engine's // memory, reset by its restart and moved by a change of words. A build that adds no account group cannot // have brought a wait, so a later build sent while the login is still owed is judged as before. if waits && !f.groupsAdded[module] { waits = false } var found []string for _, r := range h.Resources { if r.Module != module { continue } if waits && r.State == link.StateUnhealthy { continue } // **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine // left its owner and mode, and only someone at the machine can hand it over. It is no fault of this // build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict // carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy. if usedAsFound(r) && r.Since.Before(since) { found = append(found, r.Resource) continue } switch r.State { case link.StateHealthy: case link.StateStarting: return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine) case link.StateUnhealthy: if on, held := f.heldOn[module+"@"+machine]; held { return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind, r.Resource, machine, on) } return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r)) default: return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State, reasonAfter(r.Reason)) } } if waits || len(found) > 0 { var said []string if waits { said = append(said, wait) } if len(found) > 0 { said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+ "(`mesh-host hand-over ` at the machine)", machine, module, strings.Join(found, ", "))) } return healthPerson, strings.Join(said, "; ") } return healthGood, "" } // usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339). func usedAsFound(r inventory.ResourceHealth) bool { return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) } func reasonAfter(s string) string { if s == "" { return "" } return ": " + s } // healthLines is what `node show` says of a machine's long-running resources: each with its state and // since when, an unhealthy one said once marked unconfirmed. func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string { if !had { return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"} } if len(h.Resources) == 0 { return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))} } out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))} for _, r := range h.Resources { line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target, r.Since.Local().Format("2006-01-02 15:04")) if r.Reason != "" { line += " — " + r.Reason } if r.Restarts > 0 { line += fmt.Sprintf(", %d restart(s) counted", r.Restarts) } if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter { line += " (unconfirmed: said once)" } out = append(out, line) } return out } // accountGroups is every account group a manifest declares, as "/": a user resource's // groups (ADR 0252). func accountGroups(m catalogue.Manifest) map[string]bool { out := map[string]bool{} for _, r := range m.Resources { if t, _ := r["type"].(string); t != "user" { continue } name, _ := r["name"].(string) groups, _ := r["groups"].([]any) for _, g := range groups { if group, ok := g.(string); ok && group != "" { out[name+"/"+group] = true } } } return out } // addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the // earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to // the machine (no earlier manifest) adds every group it declares. func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool { before := map[string]bool{} if hadFrom { before = accountGroups(from) } for g := range accountGroups(to) { if !before[g] { return true } } return false } // kindUsedAsFound is a module's condition while the node-engine uses one of its directories as found (novox/hq // issue 339): its own kind, the operator's, never urgent, and never read by the gate as a fault of a build. const kindUsedAsFound = "directory-used-as-found" // usedAsFoundKey is a module's used-as-found condition on a machine. func usedAsFoundKey(module, node string) string { return conditions.Key(conditions.ScopeModule, module+"."+node, kindUsedAsFound) } // foundWait is whether everything unhealthy of a module on a machine is a directory used as found, and that in // one sentence. Anything else unhealthy beside it is judged as a fault, with the directory among its resources. func foundWait(module, node string, rs []inventory.ResourceHealth) (string, bool) { var ids, why []string for _, r := range rs { if r.Module != module || r.State != link.StateUnhealthy { continue } if !usedAsFound(r) { return "", false } ids = append(ids, r.Resource) why = append(why, strings.TrimSpace(strings.TrimPrefix(r.Reason, link.ReasonUsedAsFound))) } if len(ids) == 0 { return "", false } return fmt.Sprintf("%s on %s uses %s as found: %s", module, node, strings.Join(ids, ", "), strings.Join(why, "; ")), true } // usedAsFoundObservation is a module whose directory the node-engine uses as found, in words: the operator's, a // warning however long it stays, its summary naming the directories and their owners; the paths are evidence. func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation { o := moduleUnhealthyObservation(module, node, rs) o.Token, o.Kind, o.Resolver, o.Severity, o.Summary = kindUsedAsFound, kindUsedAsFound, conditions.ResolverOperator, conditions.Warning, said o.Headline = fmt.Sprintf("%s waits for a directory on %s", module, node) o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+ "The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.", module, node, module, module) o.Needs = fmt.Sprintf("on %s, run mesh-host hand-over with the directory's path as root.", node) o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node) o.Actions = nil return o }