323 lines
12 KiB
Go
323 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C).
|
|
//
|
|
// With twelve consumers of the database provision and thirty-six of a route, one provider down would be
|
|
// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's
|
|
// check names, in `needs`, the provision it exercises; while **that provision's provider for this
|
|
// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or
|
|
// the provider its credential for the provision is from — is unhealthy on the record, what the check finds
|
|
// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and
|
|
// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait.
|
|
//
|
|
// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its
|
|
// provider does; so is anything a check that names no provision finds, and anything found while the
|
|
// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not
|
|
// change that.
|
|
|
|
// holding is what one reading of the record needs to say who waits on whom: every machine's newest
|
|
// statement, the open conditions, and the catalogue — read once, asked many times.
|
|
type holding struct {
|
|
ctx context.Context
|
|
inv *inventory.Inventory
|
|
healths map[string]inventory.NodeHealth
|
|
open []conditions.Condition
|
|
shelf map[string]catalogue.Manifest
|
|
// providers memoises providerFor by machine, consumer and provision.
|
|
providers map[string]providerLookup
|
|
// waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450).
|
|
waits operatorWaitFacts
|
|
}
|
|
|
|
// checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so
|
|
// a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450).
|
|
// What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its
|
|
// check seems to wait for the operator while its unhealthy condition is open.
|
|
func (h *holding) checked(machine string) []inventory.ResourceHealth {
|
|
rs := h.healths[machine].Resources
|
|
mods := waitingModules(rs)
|
|
if len(mods) == 0 {
|
|
return rs
|
|
}
|
|
if h.waits.manifests == nil {
|
|
h.waits.manifests = map[string]catalogue.Manifest{}
|
|
}
|
|
for _, module := range mods {
|
|
if _, has := h.waits.manifests[module]; !has {
|
|
if m, ok := h.manifestOf(module); ok {
|
|
h.waits.manifests[module] = m
|
|
}
|
|
}
|
|
}
|
|
readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits)
|
|
return checkWaiting(machine, rs, h.waits)
|
|
}
|
|
|
|
type providerLookup struct {
|
|
chosen catalogue.Chosen
|
|
ok bool
|
|
}
|
|
|
|
// readHolding reads what the hold is judged from.
|
|
func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
healths, err := inv.Healths(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
|
}
|
|
|
|
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
|
// judging the record it holds under (novox/hq issue 405).
|
|
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
if inv == nil {
|
|
return nil, nil
|
|
}
|
|
return readHolding(ctx, inv, open)
|
|
}
|
|
|
|
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
|
// may be held. Down and restarting are liveness, the resource's own.
|
|
func heldFinding(r inventory.ResourceHealth) bool {
|
|
return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" &&
|
|
r.Reason != "down" && r.Reason != "restarting"
|
|
}
|
|
|
|
// providerFor is the provider composed for a consumer's provision: its recorded binding, else the
|
|
// machine its credential for the provision comes from and the module there that provides it.
|
|
func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
|
key := machine + "\x00" + consumer + "\x00" + provision
|
|
if p, known := h.providers[key]; known {
|
|
return p.chosen, p.ok
|
|
}
|
|
chosen, ok := h.lookUpProvider(machine, consumer, provision)
|
|
h.providers[key] = providerLookup{chosen, ok}
|
|
return chosen, ok
|
|
}
|
|
|
|
func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
|
if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil {
|
|
if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" {
|
|
return c, true
|
|
}
|
|
}
|
|
secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer)
|
|
if err != nil {
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
for _, s := range secrets {
|
|
if s.Name != provision || s.Provider == "" {
|
|
continue
|
|
}
|
|
if h.shelf == nil {
|
|
if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil {
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
}
|
|
assigned, err := h.inv.Assigned(h.ctx, s.Provider)
|
|
if err != nil {
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
for _, module := range assigned {
|
|
for _, offer := range h.shelf[module].Offers() {
|
|
if offer == provision {
|
|
return catalogue.Chosen{Node: s.Provider, Module: module}, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
|
|
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
|
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
|
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
|
// unknown); else healthy.
|
|
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
|
for _, c := range h.open {
|
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
|
return true, false, nil
|
|
}
|
|
}
|
|
for _, r := range h.checked(p.Node) {
|
|
if r.Module != p.Module {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateUnhealthy:
|
|
return true, false, nil
|
|
case link.StateWaiting:
|
|
waiting = true
|
|
waits = append(waits, r.Waits...)
|
|
}
|
|
}
|
|
if !waiting {
|
|
for _, c := range h.open {
|
|
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
|
}
|
|
}
|
|
return false, waiting, waits
|
|
}
|
|
|
|
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
|
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
|
if h.shelf == nil && h.inv != nil {
|
|
shelf, err := h.inv.Catalogue(h.ctx)
|
|
if err != nil {
|
|
return catalogue.Manifest{}, false
|
|
}
|
|
h.shelf = shelf
|
|
}
|
|
m, ok := h.shelf[module]
|
|
return m, ok
|
|
}
|
|
|
|
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
|
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
|
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
|
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
|
// then is not held, since holding it would hide a fault that may be its own.
|
|
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
|
credential := ""
|
|
if m, ok := h.manifestOf(p.Module); ok {
|
|
credential, _ = m.SharedCredentialOf(provision)
|
|
}
|
|
var out []inventory.Wait
|
|
for _, w := range waits {
|
|
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
|
out = append(out, w)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
|
// operator, for the part the consumer needs — the waits that hold it.
|
|
type heldUnderProvider struct {
|
|
provider catalogue.Chosen
|
|
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
|
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
|
waits []inventory.Wait
|
|
}
|
|
|
|
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
|
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
|
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
|
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
|
by, held := h.heldWith(machine, module, rs)
|
|
return by.provider, held
|
|
}
|
|
|
|
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
|
var on heldUnderProvider
|
|
onlyWaits := true
|
|
for _, r := range rs {
|
|
if r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
if !heldFinding(r) {
|
|
return heldUnderProvider{}, false
|
|
}
|
|
p, ok := h.providerFor(machine, module, r.Needs)
|
|
if !ok || (p.Node == machine && p.Module == module) {
|
|
return heldUnderProvider{}, false
|
|
}
|
|
unhealthy, waiting, waits := h.providerState(p)
|
|
switch {
|
|
case unhealthy:
|
|
onlyWaits = false
|
|
case waiting:
|
|
covered := h.covering(p, r.Needs, waits)
|
|
if len(covered) == 0 {
|
|
return heldUnderProvider{}, false
|
|
}
|
|
on.waits = append(on.waits, covered...)
|
|
default:
|
|
return heldUnderProvider{}, false
|
|
}
|
|
on.provider = p
|
|
}
|
|
if !onlyWaits {
|
|
on.waits = nil
|
|
}
|
|
return on, on.provider.Module != ""
|
|
}
|
|
|
|
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
|
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
|
// the provider waits, so neither is hidden.
|
|
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
|
for _, r := range rs {
|
|
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
|
continue
|
|
}
|
|
p, ok := h.providerFor(machine, module, r.Needs)
|
|
if !ok || (p.Node == machine && p.Module == module) {
|
|
continue
|
|
}
|
|
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
|
return p, true
|
|
}
|
|
}
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
|
|
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
|
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
|
var out []string
|
|
for machine, nh := range h.healths {
|
|
byModule := map[string][]inventory.ResourceHealth{}
|
|
for _, r := range nh.Resources {
|
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
|
byModule[r.Module] = append(byModule[r.Module], r)
|
|
}
|
|
}
|
|
for module, rs := range byModule {
|
|
if on, held := h.heldUnder(machine, module, rs); held && on == p {
|
|
out = append(out, module+" on "+machine)
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
|
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
|
out := map[string]heldUnderProvider{}
|
|
byModule := map[string][]inventory.ResourceHealth{}
|
|
for _, r := range h.healths[machine].Resources {
|
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
|
byModule[r.Module] = append(byModule[r.Module], r)
|
|
}
|
|
}
|
|
for module, rs := range byModule {
|
|
if on, held := h.heldWith(machine, module, rs); held {
|
|
out[module] = on
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// moduleUnhealthyKey is a module's health condition's key on a machine.
|
|
func moduleUnhealthyKey(module, machine string) string {
|
|
return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy"
|
|
}
|
|
|
|
// waitingWords is the provider's evidence that consumers wait on it.
|
|
func waitingWords(waiters []string) string {
|
|
return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", "))
|
|
}
|