jschoubben
ed45cc6415
A provider whose wait fails the check raises unhealthy, but the holding read its stored statement with the wait unchecked: sayWaiters built the needs-operator key, found it not open and listed no held consumer. The holding now reads each statement as judged (ADR 0283 decision 3), in sayWaiters and in the provider's state its consumers are held by.
323 lines
12 KiB
Go
323 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C).
|
|
//
|
|
// With twelve consumers of the database provision and thirty-six of a route, one provider down would be
|
|
// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's
|
|
// check names, in `needs`, the provision it exercises; while **that provision's provider for this
|
|
// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or
|
|
// the provider its credential for the provision is from — is unhealthy on the record, what the check finds
|
|
// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and
|
|
// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait.
|
|
//
|
|
// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its
|
|
// provider does; so is anything a check that names no provision finds, and anything found while the
|
|
// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not
|
|
// change that.
|
|
|
|
// holding is what one reading of the record needs to say who waits on whom: every machine's newest
|
|
// statement, the open conditions, and the catalogue — read once, asked many times.
|
|
type holding struct {
|
|
ctx context.Context
|
|
inv *inventory.Inventory
|
|
healths map[string]inventory.NodeHealth
|
|
open []conditions.Condition
|
|
shelf map[string]catalogue.Manifest
|
|
// providers memoises providerFor by machine, consumer and provision.
|
|
providers map[string]providerLookup
|
|
// waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450).
|
|
waits operatorWaitFacts
|
|
}
|
|
|
|
// checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so
|
|
// a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450).
|
|
// What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its
|
|
// check seems to wait for the operator while its unhealthy condition is open.
|
|
func (h *holding) checked(machine string) []inventory.ResourceHealth {
|
|
rs := h.healths[machine].Resources
|
|
mods := waitingModules(rs)
|
|
if len(mods) == 0 {
|
|
return rs
|
|
}
|
|
if h.waits.manifests == nil {
|
|
h.waits.manifests = map[string]catalogue.Manifest{}
|
|
}
|
|
for _, module := range mods {
|
|
if _, has := h.waits.manifests[module]; !has {
|
|
if m, ok := h.manifestOf(module); ok {
|
|
h.waits.manifests[module] = m
|
|
}
|
|
}
|
|
}
|
|
readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits)
|
|
return checkWaiting(machine, rs, h.waits)
|
|
}
|
|
|
|
type providerLookup struct {
|
|
chosen catalogue.Chosen
|
|
ok bool
|
|
}
|
|
|
|
// readHolding reads what the hold is judged from.
|
|
func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
healths, err := inv.Healths(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
|
}
|
|
|
|
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
|
// judging the record it holds under (novox/hq issue 405).
|
|
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
if inv == nil {
|
|
return nil, nil
|
|
}
|
|
return readHolding(ctx, inv, open)
|
|
}
|
|
|
|
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
|
// may be held. Down and restarting are liveness, the resource's own.
|
|
func heldFinding(r inventory.ResourceHealth) bool {
|
|
return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" &&
|
|
r.Reason != "down" && r.Reason != "restarting"
|
|
}
|
|
|
|
// providerFor is the provider composed for a consumer's provision: its recorded binding, else the
|
|
// machine its credential for the provision comes from and the module there that provides it.
|
|
func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
|
key := machine + "\x00" + consumer + "\x00" + provision
|
|
if p, known := h.providers[key]; known {
|
|
return p.chosen, p.ok
|
|
}
|
|
chosen, ok := h.lookUpProvider(machine, consumer, provision)
|
|
h.providers[key] = providerLookup{chosen, ok}
|
|
return chosen, ok
|
|
}
|
|
|
|
func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) {
|
|
if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil {
|
|
if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" {
|
|
return c, true
|
|
}
|
|
}
|
|
secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer)
|
|
if err != nil {
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
for _, s := range secrets {
|
|
if s.Name != provision || s.Provider == "" {
|
|
continue
|
|
}
|
|
if h.shelf == nil {
|
|
if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil {
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
}
|
|
assigned, err := h.inv.Assigned(h.ctx, s.Provider)
|
|
if err != nil {
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
for _, module := range assigned {
|
|
for _, offer := range h.shelf[module].Offers() {
|
|
if offer == provision {
|
|
return catalogue.Chosen{Node: s.Provider, Module: module}, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
|
|
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
|
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
|
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
|
// unknown); else healthy.
|
|
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
|
for _, c := range h.open {
|
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
|
return true, false, nil
|
|
}
|
|
}
|
|
for _, r := range h.checked(p.Node) {
|
|
if r.Module != p.Module {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateUnhealthy:
|
|
return true, false, nil
|
|
case link.StateWaiting:
|
|
waiting = true
|
|
waits = append(waits, r.Waits...)
|
|
}
|
|
}
|
|
if !waiting {
|
|
for _, c := range h.open {
|
|
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
|
}
|
|
}
|
|
return false, waiting, waits
|
|
}
|
|
|
|
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
|
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
|
if h.shelf == nil && h.inv != nil {
|
|
shelf, err := h.inv.Catalogue(h.ctx)
|
|
if err != nil {
|
|
return catalogue.Manifest{}, false
|
|
}
|
|
h.shelf = shelf
|
|
}
|
|
m, ok := h.shelf[module]
|
|
return m, ok
|
|
}
|
|
|
|
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
|
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
|
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
|
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
|
// then is not held, since holding it would hide a fault that may be its own.
|
|
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
|
credential := ""
|
|
if m, ok := h.manifestOf(p.Module); ok {
|
|
credential, _ = m.SharedCredentialOf(provision)
|
|
}
|
|
var out []inventory.Wait
|
|
for _, w := range waits {
|
|
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
|
out = append(out, w)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
|
// operator, for the part the consumer needs — the waits that hold it.
|
|
type heldUnderProvider struct {
|
|
provider catalogue.Chosen
|
|
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
|
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
|
waits []inventory.Wait
|
|
}
|
|
|
|
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
|
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
|
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
|
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
|
by, held := h.heldWith(machine, module, rs)
|
|
return by.provider, held
|
|
}
|
|
|
|
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
|
var on heldUnderProvider
|
|
onlyWaits := true
|
|
for _, r := range rs {
|
|
if r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
if !heldFinding(r) {
|
|
return heldUnderProvider{}, false
|
|
}
|
|
p, ok := h.providerFor(machine, module, r.Needs)
|
|
if !ok || (p.Node == machine && p.Module == module) {
|
|
return heldUnderProvider{}, false
|
|
}
|
|
unhealthy, waiting, waits := h.providerState(p)
|
|
switch {
|
|
case unhealthy:
|
|
onlyWaits = false
|
|
case waiting:
|
|
covered := h.covering(p, r.Needs, waits)
|
|
if len(covered) == 0 {
|
|
return heldUnderProvider{}, false
|
|
}
|
|
on.waits = append(on.waits, covered...)
|
|
default:
|
|
return heldUnderProvider{}, false
|
|
}
|
|
on.provider = p
|
|
}
|
|
if !onlyWaits {
|
|
on.waits = nil
|
|
}
|
|
return on, on.provider.Module != ""
|
|
}
|
|
|
|
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
|
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
|
// the provider waits, so neither is hidden.
|
|
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
|
for _, r := range rs {
|
|
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
|
continue
|
|
}
|
|
p, ok := h.providerFor(machine, module, r.Needs)
|
|
if !ok || (p.Node == machine && p.Module == module) {
|
|
continue
|
|
}
|
|
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
|
return p, true
|
|
}
|
|
}
|
|
return catalogue.Chosen{}, false
|
|
}
|
|
|
|
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
|
func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
|
var out []string
|
|
for machine, nh := range h.healths {
|
|
byModule := map[string][]inventory.ResourceHealth{}
|
|
for _, r := range nh.Resources {
|
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
|
byModule[r.Module] = append(byModule[r.Module], r)
|
|
}
|
|
}
|
|
for module, rs := range byModule {
|
|
if on, held := h.heldUnder(machine, module, rs); held && on == p {
|
|
out = append(out, module+" on "+machine)
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
|
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
|
out := map[string]heldUnderProvider{}
|
|
byModule := map[string][]inventory.ResourceHealth{}
|
|
for _, r := range h.healths[machine].Resources {
|
|
if r.Module != "" && r.State == link.StateUnhealthy {
|
|
byModule[r.Module] = append(byModule[r.Module], r)
|
|
}
|
|
}
|
|
for module, rs := range byModule {
|
|
if on, held := h.heldWith(machine, module, rs); held {
|
|
out[module] = on
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// moduleUnhealthyKey is a module's health condition's key on a machine.
|
|
func moduleUnhealthyKey(module, machine string) string {
|
|
return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy"
|
|
}
|
|
|
|
// waitingWords is the provider's evidence that consumers wait on it.
|
|
func waitingWords(waiters []string) string {
|
|
return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", "))
|
|
}
|